8000 Merge branch 'main' into patch-3 · tebjan/llama-cpp-python@32cd7e4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 32cd7e4

Browse files
abetlenabetlen
authored
Merge branch 'main' into patch-3
2 parents b5c3227 + c68e7fb commit 32cd7e4

File tree

4 files changed

+200
-64
lines changed

4 files changed

+200
-64
lines changed

‎.github/workflows/generate-index-from-release.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ jobs:
3535
- name: Setup Pages
3636
uses: actions/configure-pages@v5
3737
- name: Build
38+
env:
39+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3840
run: |
41+
./scripts/get-releases.sh
3942
./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
4043
./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
4144
./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'

‎llama_cpp/llama_chat_format.py

Lines changed: 67 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2707,6 +2707,31 @@ def last_image_embed_free():
27072707
def load_image(self, image_url: str) -> bytes:
27082708
return self._load_image(image_url)
27092709

2710+
def _embed_image_bytes(self, image_bytes: bytes, n_threads_batch: int = 1):
2711+
if (
2712+
self._last_image_embed is not None
2713+
and self._last_image_hash is not None
2714+
and hash(image_bytes) == self._last_image_hash
2715+
):
2716+
return self._last_image_embed
2717+
with suppress_stdout_stderr(disable=self.verbose):
2718+
# Free the previous image embed
2719+
if self._last_image_embed is not None:
2720+
self._llava_cpp.llava_image_embed_free(self._last_image_embed)
2721+
self._last_image_embed = None
2722+
self._last_image_hash = None
2723+
embed = self._llava_cpp.llava_image_embed_make_with_bytes(
2724+
self.clip_ctx,
2725+
n_threads_batch,
2726+
(ctypes.c_uint8 * len(image_bytes)).from_buffer(
2727+
bytearray(image_bytes)
2728+
),
2729+
len(image_bytes),
2730+
)
2731+
self._last_image_embed = embed
2732+
self._last_image_hash = hash(image_bytes)
2733+
return embed
2734+
27102735
def __call__(
27112736
self,
27122737
*,
@@ -2769,30 +2794,9 @@ def __call__(
27692794
)
27702795
split_text = self.split_text_on_image_urls(text, image_urls)
27712796

2772-
def embed_image_bytes(image_bytes: bytes):
2773-
if (
2774-
self._last_image_embed is not None
2775-
and self._last_image_hash is not None
2776-
and hash(image_bytes) == self._last_image_hash
2777-
):
2778-
return self._last_image_embed
2779-
with suppress_stdout_stderr(disable=self.verbose):
2780-
# Free the previous image embed
2781-
if self._last_image_embed is not None:
2782-
self._llava_cpp.llava_image_embed_free(self._last_image_embed)
2783-
self._last_image_embed = None
2784-
self._last_image_hash = None
2785-
embed = self._llava_cpp.llava_image_embed_make_with_bytes(
2786-
self.clip_ctx,
2787-
llama.context_params.n_threads_batch,
2788-
(ctypes.c_uint8 * len(image_bytes)).from_buffer(
2789-
bytearray(image_bytes)
2790-
),
2791-
len(image_bytes),
2792-
)
2793-
self._last_image_embed = embed
2794-
self._last_image_hash = hash(image_bytes)
2795-
return embed
2797+
if self.verbose:
2798+
print(text, file=sys.stderr)
2799+
27962800

27972801
# Evaluate prompt
27982802
llama.reset()
@@ -2809,7 +2813,7 @@ def embed_image_bytes(image_bytes: bytes):
28092813
llama.eval(tokens)
28102814
else:
28112815
image_bytes = self.load_image(value)
2812-
embed = embed_image_bytes(image_bytes)
2816+
embed = self._embed_image_bytes(image_bytes, llama.context_params.n_threads_batch)
28132817
if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx():
28142818
raise ValueError(
28152819
f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}"
@@ -3308,6 +3312,44 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler):
33083312
Llama3VisionAlpha = Llama3VisionAlphaChatHandler
33093313

33103314

3315+
class MiniCPMv26ChatHandler(Llava15ChatHandler):
3316+
DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant."
3317+
3318+
CHAT_FORMAT = (
3319+
"{% for message in messages %}"
3320+
"{% if loop.first and messages[0]['role'] != 'system' %}"
3321+
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
3322+
"{% endif %}"
3323+
"<|im_start|>{{ message['role'] }}\n"
3324+
"{% if message['content'] is iterable %}"
3325+
"{% for content in message['content'] %}"
3326+
"{% if content.type == 'image_url' %}"
3327+
"{% if content.image_url is string %}"
3328+
"{{ content.image_url }}"
3329+
"{% endif %}"
3330+
"{% if content.image_url is mapping %}"
3331+
"{{ content.image_url.url }}"
3332+
"{% endif %}"
3333+
"{% endif %}"
3334+
"{% endfor %}"
3335+
3336+
"{% for content in message['content'] %}"
3337+
"{% if content.type == 'text' %}"
3338+
"{{ content.text }}"
3339+
"{% endif %}"
3340+
"{% endfor %}"
3341+
"{% endif %}"
3342+
"{% if message['content'] is string %}"
3343+
"{{ message['content'] }}"
3344+
"{% endif %}"
3345+
"<|im_end|>\n"
3346+
"{% endfor %}"
3347+
"{% if add_generation_prompt %}"
3348+
"<|im_start|>assistant\n"
3349+
"{% endif %}"
3350+
)
3351+
3352+
33113353
@register_chat_completion_handler("chatml-function-calling")
33123354
def chatml_function_calling(
33133355
llama: llama.Llama,

‎scripts/get-releases.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Function to get all releases
4+
get_all_releases() {
5+
local page=1
6+
local per_page=100
7+
local releases=""
8+
local new_releases
9+
10+
# Prepare headers
11+
local headers=(-H "Accept: application/vnd.github.v3+json")
12+
if [ -n "$GITHUB_TOKEN" ]; then
13+
headers+=(-H "Authorization: Bearer $GITHUB_TOKEN")
14+
fi
15+
16+
while true; do
17+
response=$(curl -s "${headers[@]}" \
18+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases?page=$page&per_page=$per_page")
19+
20+
# Check if the response is valid JSON
21+
if ! echo "$response" | jq empty > /dev/null 2>&1; then
22+
echo "Error: Invalid response from GitHub API" >&2
23+
echo "Response: $response" >&2
24+
return 1
25+
fi
26+
27+
new_releases=$(echo "$response" | jq -r '.[].tag_name')
28+
if [ -z "$new_releases" ]; then
29+
break
30+
fi
31+
releases="$releases $new_releases"
32+
((page++))
33+
done
34+
35+
echo $releases
36+
}
37+
38+
# Get all releases and save to file
39+
releases=$(get_all_releases)
40+
if [ $? -ne 0 ]; then
41+
echo "Failed to fetch releases. Please check your internet connection and try again later." >&2
42+
exit 1
43+
fi
44+
45+
echo "$releases" | tr ' ' '\n' > all_releases.txt
46+
47+
echo "All releases have been saved to all_releases.txt"

‎scripts/releases-to-pep-503.sh

Lines changed: 83 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,104 @@
11
#!/bin/bash
22

3+
# Enable exit on error
4+
set -e
5+
6+
# Function for logging
7+
log_error() {
8+
echo "ERROR: $1" >&2
9+
}
10+
11+
log_info() {
12+
echo "INFO: $1"
13+
}
14+
315
# Get output directory or default to index/whl/cpu
416
output_dir=${1:-"index/whl/cpu"}
517

6-
# Create output directory
7-
mkdir -p $output_dir
8-
9-
# Change to output directory
10-
pushd $output_dir
18+
# Get pattern from second arg or default to valid python package version pattern
19+
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
1120

12-
# Create an index html file
13-
echo "<!DOCTYPE html>" > index.html
14-
echo "<html>" >> index.html
15-
echo " <head></head>" >> index.html
16-
echo " <body>" >> index.html
17-
echo " <a href=\"llama-cpp-python/\">llama-cpp-python</a>" >> index.html
18-
echo " <br>" >> index.html
19-
echo " </body>" >> index.html
20-
echo "</html>" >> index.html
21-
echo "" >> index.html
21+
# Get the current directory (where the script is run from)
22+
current_dir="$(pwd)"
2223

23-
# Create llama-cpp-python directory
24-
mkdir -p llama-cpp-python
24+
# Check if all_releases.txt exists
25+
if [ ! -f "$current_dir/all_releases.txt" ]; then
26+
log_error "all_releases.txt not found in the current directory."
27+
exit 1
28+
fi
2529

26-
# Change to llama-cpp-python directory
27-
pushd llama-cpp-python
30+
# Create output directory
31+
mkdir -p "$output_dir"
2832

2933
# Create an index html file
30-
echo "<!DOCTYPE html>" > index.html
31-
echo "<html>" >> index.html
32-
echo " <body>" >> index.html
33-
echo " <h1>Links for llama-cpp-python</h1>" >> index.html
34+
cat << EOF > "$output_dir/index.html"
35+
<!DOCTYPE html>
36+
<html>
37+
<head></head>
38+
<body>
39+
<a href="llama-cpp-python/">llama-cpp-python</a>
40+
<br>
41+
</body>
42+
</html>
3443
35-
# Get all releases
36-
releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name)
44+
EOF
3745

38-
# Get pattern from second arg or default to valid python package version pattern
39-
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
46+
# Create llama-cpp-python directory
47+
mkdir -p "$output_dir/llama-cpp-python"
48+
49+
# Create an index html file in llama-cpp-python directory
50+
cat << EOF > "$output_dir/llama-cpp-python/index.html"
51+
<!DOCTYPE html>
52+
<html>
53+
<body>
54+
<h1>Links for llama-cpp-python</h1>
55+
EOF
4056

4157
# Filter releases by pattern
42-
releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern)
58+
releases=$(grep -E "$pattern" "$current_dir/all_releases.txt")
59+
60+
# Prepare curl headers
61+
headers=('--header' 'Accept: application/vnd.github.v3+json')
62+
if [ -n "$GITHUB_TOKEN" ]; then
63+
headers+=('--header' "authorization: Bearer $GITHUB_TOKEN")
64+
fi
65+
headers+=('--header' 'content-type: application/json')
4366

4467
# For each release, get all assets
4568
for release in $releases; do
46-
assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets)
69+
log_info "Processing release: $release"
70+
response=$(curl -s "${headers[@]}" \
71+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release")
72+
73+
if [ -z "$response" ]; then
74+
log_error "Empty response from GitHub API for release $release"
75+
continue
76+
fi
77+
78+
if ! echo "$response" | jq -e '.assets' > /dev/null 2>&1; then
79+
log_error "Invalid or unexpected response from GitHub API for release $release"
80+
log_error "Response: $response"
81+
continue
82+
fi
83+
4784
# Get release version from release ie v0.1.0-cu121 -> v0.1.0
48-
release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
49-
echo " <h2>$release_version</h2>" >> index.html
50-
for asset in $(echo $assets | jq -r .[].browser_download_url); do
51-
if [[ $asset == *".whl" ]]; then
52-
echo " <a href=\"$asset\">$asset</a>" >> index.html
53-
echo " <br>" >> index.html
54-
fi
85+
release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
86+
echo " <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html"
87+
88+
wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url')
89+
if [ -z "$wheel_urls" ]; then
90+
log_error "No wheel files found for release $release"
91+
continue
92+
fi
93+
94+
echo "$wheel_urls" | while read -r asset; do
95+
echo " <a href=\"$asset\">$asset</a>" >> "$output_dir/llama-cpp-python/index.html"
96+
echo " <br>" >> "$output_dir/llama-cpp-python/index.html"
5597
done
5698
done
5799

58-
echo " </body>" >> index.html
59-
echo "</html>" >> index.html
60-
echo "" >> index.html
100+
echo " </body>" >> "$output_dir/llama-cpp-python/index.html"
101+
echo "</html>" >> "$output_dir/llama-cpp-python/index.html"
102+
echo "" >> "$output_dir/llama-cpp-python/index.html"
103+
104+
log_info "Index generation complete. Output directory: $output_dir"

0 commit comments

Comments
 (0)
0