From 6b9bdf8ad80e68c35fd6e6e4834ee63395f9596d Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Wed, 25 Jun 2025 16:35:34 -0500 Subject: [PATCH 1/7] add tar support for lambdas --- bin-functions/install-native.sh | 15 ++++++- scripts/sock_test.py | 2 +- scripts/test.py | 36 ++++++++++----- src/worker/lambda/handlerPuller.go | 37 +++------------ test-registry/echo.tar.gz | Bin 0 -> 170 bytes test-registry/echo/f.py | 2 - test-registry/fbomb.tar.gz | Bin 0 -> 460 bytes test-registry/fbomb/f.py | 30 ------------- test-registry/flask-test.tar.gz | Bin 0 -> 713 bytes test-registry/flask-test/f.py | 14 ------ test-registry/flask-test/requirements.in | 2 - test-registry/flask-test/requirements.txt | 24 ---------- test-registry/hello.tar.gz | Bin 0 -> 176 bytes test-registry/hello/f.py | 2 - test-registry/hello2.tar.gz | Bin 0 -> 176 bytes test-registry/hello2/f.py | 2 - test-registry/install1.tar.gz | Bin 0 -> 408 bytes test-registry/install1/f.py | 5 --- test-registry/install1/requirements.in | 1 - test-registry/install1/requirements.txt | 16 ------- test-registry/install2.tar.gz | Bin 0 -> 430 bytes test-registry/install2/f.py | 5 --- test-registry/install2/requirements.in | 5 --- test-registry/install2/requirements.txt | 24 ---------- test-registry/install3.tar.gz | Bin 0 -> 434 bytes test-registry/install3/f.py | 6 --- test-registry/install3/requirements.in | 2 - test-registry/install3/requirements.txt | 18 -------- test-registry/lambda-config-test.tar.gz | Bin 0 -> 808 bytes test-registry/lambda-config-test/f.py | 14 ------ test-registry/lambda-config-test/ol.yaml | 5 --- .../lambda-config-test/requirements.in | 2 - .../lambda-config-test/requirements.txt | 24 ---------- test-registry/max_mem_alloc.tar.gz | Bin 0 -> 473 bytes test-registry/max_mem_alloc/f.py | 30 ------------- test-registry/numpy21.tar.gz | Bin 0 -> 395 bytes test-registry/numpy21/f.py | 7 --- test-registry/numpy21/requirements.in | 1 - test-registry/numpy21/requirements.txt | 8 ---- test-registry/numpy22.tar.gz | Bin 0 -> 395 bytes test-registry/numpy22/f.py | 7 --- test-registry/numpy22/requirements.in | 1 - test-registry/numpy22/requirements.txt | 8 ---- test-registry/pandas-tests.tar.gz | Bin 0 -> 1041 bytes test-registry/pandas-tests/f.py | 42 ------------------ test-registry/pandas-tests/requirements.in | 4 -- test-registry/pandas-tests/requirements.txt | 39 ---------------- test-registry/pandas-v1.tar.gz | Bin 0 -> 474 bytes test-registry/pandas-v1/f.py | 6 --- test-registry/pandas-v1/requirements.in | 2 - test-registry/pandas-v1/requirements.txt | 18 -------- test-registry/pandas.tar.gz | Bin 0 -> 473 bytes test-registry/pandas/f.py | 6 --- test-registry/pandas/requirements.in | 1 - test-registry/pandas/requirements.txt | 18 -------- test-registry/server.tar.gz | Bin 0 -> 203 bytes test-registry/server/echo.py | 2 - test-registry/server/hello.py | 2 - test-registry/syslog.tar.gz | Bin 0 -> 260 bytes test-registry/syslog/f.py | 6 --- test-registry/thread_counter.tar.gz | Bin 0 -> 403 bytes test-registry/thread_counter/f.py | 27 ----------- test-registry/timeout.tar.gz | Bin 0 -> 218 bytes test-registry/timeout/f.py | 5 --- test-registry/version.tar.gz | Bin 0 -> 325 bytes test-registry/version/f.py | 16 ------- 66 files changed, 47 insertions(+), 502 deletions(-) create mode 100644 test-registry/echo.tar.gz delete mode 100644 test-registry/echo/f.py create mode 100644 test-registry/fbomb.tar.gz delete mode 100644 test-registry/fbomb/f.py create mode 100644 test-registry/flask-test.tar.gz delete mode 100644 test-registry/flask-test/f.py delete mode 100644 test-registry/flask-test/requirements.in delete mode 100644 test-registry/flask-test/requirements.txt create mode 100644 test-registry/hello.tar.gz delete mode 100644 test-registry/hello/f.py create mode 100644 test-registry/hello2.tar.gz delete mode 100644 test-registry/hello2/f.py create mode 100644 test-registry/install1.tar.gz delete mode 100644 test-registry/install1/f.py delete mode 100644 test-registry/install1/requirements.in delete mode 100644 test-registry/install1/requirements.txt create mode 100644 test-registry/install2.tar.gz delete mode 100644 test-registry/install2/f.py delete mode 100644 test-registry/install2/requirements.in delete mode 100644 test-registry/install2/requirements.txt create mode 100644 test-registry/install3.tar.gz delete mode 100644 test-registry/install3/f.py delete mode 100644 test-registry/install3/requirements.in delete mode 100644 test-registry/install3/requirements.txt create mode 100644 test-registry/lambda-config-test.tar.gz delete mode 100644 test-registry/lambda-config-test/f.py delete mode 100644 test-registry/lambda-config-test/ol.yaml delete mode 100644 test-registry/lambda-config-test/requirements.in delete mode 100644 test-registry/lambda-config-test/requirements.txt create mode 100644 test-registry/max_mem_alloc.tar.gz delete mode 100644 test-registry/max_mem_alloc/f.py create mode 100644 test-registry/numpy21.tar.gz delete mode 100644 test-registry/numpy21/f.py delete mode 100644 test-registry/numpy21/requirements.in delete mode 100644 test-registry/numpy21/requirements.txt create mode 100644 test-registry/numpy22.tar.gz delete mode 100644 test-registry/numpy22/f.py delete mode 100644 test-registry/numpy22/requirements.in delete mode 100644 test-registry/numpy22/requirements.txt create mode 100644 test-registry/pandas-tests.tar.gz delete mode 100644 test-registry/pandas-tests/f.py delete mode 100644 test-registry/pandas-tests/requirements.in delete mode 100644 test-registry/pandas-tests/requirements.txt create mode 100644 test-registry/pandas-v1.tar.gz delete mode 100644 test-registry/pandas-v1/f.py delete mode 100644 test-registry/pandas-v1/requirements.in delete mode 100644 test-registry/pandas-v1/requirements.txt create mode 100644 test-registry/pandas.tar.gz delete mode 100644 test-registry/pandas/f.py delete mode 100644 test-registry/pandas/requirements.in delete mode 100644 test-registry/pandas/requirements.txt create mode 100644 test-registry/server.tar.gz delete mode 100644 test-registry/server/echo.py delete mode 100644 test-registry/server/hello.py create mode 100644 test-registry/syslog.tar.gz delete mode 100644 test-registry/syslog/f.py create mode 100644 test-registry/thread_counter.tar.gz delete mode 100644 test-registry/thread_counter/f.py create mode 100644 test-registry/timeout.tar.gz delete mode 100644 test-registry/timeout/f.py create mode 100644 test-registry/version.tar.gz delete mode 100644 test-registry/version/f.py diff --git a/bin-functions/install-native.sh b/bin-functions/install-native.sh index da9a5f533..b28386f89 100644 --- a/bin-functions/install-native.sh +++ b/bin-functions/install-native.sh @@ -14,7 +14,18 @@ for f in ${NATIVE_PREFIX}/*; do # Ignore subdirectories, libraries, and non-executable files if [[ $name != *".so" && -f "$f" && -x "$f" ]]; then - echo "Installing native function '$name.bin' from '$f' to ${REGISTRY_PATH}/${name}.bin" - rsync -c $f ${REGISTRY_PATH}/$name.bin + echo "Installing native function '$name.tar.gz' from '$f' to ${REGISTRY_PATH}/${name}.tar.gz" + + # Create temporary directory for tar.gz creation + temp_dir=$(mktemp -d) + + # Copy binary to temp directory as f.bin + cp "$f" "$temp_dir/f.bin" + + # Create tar.gz file containing f.bin + tar -czf "${REGISTRY_PATH}/${name}.tar.gz" -C "$temp_dir" f.bin + + # Clean up temporary directory + rm -rf "$temp_dir" fi done diff --git a/scripts/sock_test.py b/scripts/sock_test.py index c56ed37bc..c653bdf84 100755 --- a/scripts/sock_test.py +++ b/scripts/sock_test.py @@ -39,7 +39,7 @@ def sock_churn(baseline, procs, seconds, fork): # baseline: how many sandboxes are sitting idly throughout the experiment # procs: how many procs are concurrently creating and deleting other sandboxes - echo_path = os.path.abspath("test-registry/echo") + echo_path = os.path.abspath("test-registry/echo.tar.gz") open_lambda = OpenLambda() if fork: diff --git a/scripts/test.py b/scripts/test.py index 035581f7e..ed3ac43ee 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -10,6 +10,7 @@ import os import sys import tempfile +import tarfile from time import time from subprocess import call @@ -141,13 +142,21 @@ def call_each_once_exec(lambda_count, alloc_mb, zygote_provider): def call_each_once(lambda_count, alloc_mb=0, zygote_provider="tree"): with tempfile.TemporaryDirectory() as reg_dir: - # create dummy lambdas + # create dummy lambdas as tar.gz files for pos in range(lambda_count): - with open(os.path.join(reg_dir, f"L{pos}.py"), "w", encoding='utf-8') as code: - code.write("def f(event):\n") - code.write(" global s\n") - code.write(f" s = '*' * {alloc_mb} * 1024**2\n") - code.write(f" return {pos}\n") + # Create temporary directory for lambda contents + with tempfile.TemporaryDirectory() as lambda_dir: + # Write f.py file + with open(os.path.join(lambda_dir, "f.py"), "w", encoding='utf-8') as code: + code.write("def f(event):\n") + code.write(" global s\n") + code.write(f" s = '*' * {alloc_mb} * 1024**2\n") + code.write(f" return {pos}\n") + + # Create tar.gz file + tar_path = os.path.join(reg_dir, f"L{pos}.tar.gz") + with tarfile.open(tar_path, "w:gz") as tar: + tar.add(os.path.join(lambda_dir, "f.py"), arcname="f.py") with TestConfContext(registry=reg_dir): call_each_once_exec(lambda_count=lambda_count, alloc_mb=alloc_mb, @@ -194,10 +203,17 @@ def update_code(): open_lambda = OpenLambda() for pos in range(3): - # update function code - with open(os.path.join(reg_dir, "version.py"), "w", encoding='utf-8') as code: - code.write("def f(event):\n") - code.write(f" return {pos}\n") + # update function code in tar.gz format + with tempfile.TemporaryDirectory() as lambda_dir: + # Write f.py file + with open(os.path.join(lambda_dir, "f.py"), "w", encoding='utf-8') as code: + code.write("def f(event):\n") + code.write(f" return {pos}\n") + + # Create tar.gz file + tar_path = os.path.join(reg_dir, "version.tar.gz") + with tarfile.open(tar_path, "w:gz") as tar: + tar.add(os.path.join(lambda_dir, "f.py"), arcname="f.py") # how long does it take for us to start seeing the latest code? start = time() diff --git a/src/worker/lambda/handlerPuller.go b/src/worker/lambda/handlerPuller.go index 466d1e81a..34b5c6e07 100644 --- a/src/worker/lambda/handlerPuller.go +++ b/src/worker/lambda/handlerPuller.go @@ -113,11 +113,9 @@ func (cp *HandlerPuller) Pull(name string) (rt_type common.RuntimeType, targetDi } if cp.isRemote() { - // registry type = web + // registry type = web - only support tar.gz files urls := []string{ cp.prefix + "/" + name + ".tar.gz", - cp.prefix + "/" + name + ".py", - cp.prefix + "/" + name + ".bin", } for i := 0; i < len(urls); i++ { @@ -133,12 +131,9 @@ func (cp *HandlerPuller) Pull(name string) (rt_type common.RuntimeType, targetDi return rt_type, "", fmt.Errorf("lambda not found at any of these locations: %s", strings.Join(urls, ", ")) } - // registry type = file + // registry type = file - only support tar.gz files paths := []string{ filepath.Join(cp.prefix, name) + ".tar.gz", - filepath.Join(cp.prefix, name) + ".py", - filepath.Join(cp.prefix, name) + ".bin", - filepath.Join(cp.prefix, name), } for i := 0; i < len(paths); i++ { @@ -208,26 +203,8 @@ func (cp *HandlerPuller) pullLocalFile(src, lambdaName string) (rt_type common.R log.Printf("Created new directory for lambda function at `%s`", targetDir) - // Make sure we include the suffix - if strings.HasSuffix(stat.Name(), ".py") { - log.Printf("Installing `%s` from a python file", src) - - err := Copy(src, filepath.Join(targetDir, "f.py")) - rt_type = common.RT_PYTHON - - if err != nil { - return rt_type, "", fmt.Errorf("%s :: %s", err) - } - } else if strings.HasSuffix(stat.Name(), ".bin") { - log.Printf("Installing `%s` from binary file", src) - - err := Copy(src, filepath.Join(targetDir, "f.bin")) - rt_type = common.RT_NATIVE - - if err != nil { - return rt_type, "", fmt.Errorf("%s :: %s", err) - } - } else if strings.HasSuffix(stat.Name(), ".tar.gz") { + // Only support tar.gz files + if strings.HasSuffix(stat.Name(), ".tar.gz") { log.Printf("Installing `%s` from an archive file", src) cmd := exec.Command("tar", "-xzf", src, "--directory", targetDir) @@ -235,16 +212,16 @@ func (cp *HandlerPuller) pullLocalFile(src, lambdaName string) (rt_type common.R return rt_type, "", fmt.Errorf("%s :: %s", err, string(output)) } - // Figure out runtime type + // Validate that tar.gz contains f.py or f.bin if _, err := os.Stat(targetDir + "/f.py"); !os.IsNotExist(err) { rt_type = common.RT_PYTHON } else if _, err := os.Stat(targetDir + "/f.bin"); !os.IsNotExist(err) { rt_type = common.RT_NATIVE } else { - return rt_type, "", fmt.Errorf("Found unknown runtime type or no code at all") + return rt_type, "", fmt.Errorf("tar.gz file must contain f.py or f.bin") } } else { - return rt_type, "", fmt.Errorf("lambda file %s not a .tar.gz or .py", src) + return rt_type, "", fmt.Errorf("lambda file %s must be a .tar.gz file", src) } if !cp.isRemote() { diff --git a/test-registry/echo.tar.gz b/test-registry/echo.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f6e6ab268eb10c75d2974d99dae51ef9bb272f5e GIT binary patch literal 170 zcmb2|=3oE==C>Cexegf!v_6ddE)w}_nQxXlWB%e9dfiuCK78BOrO;F=>hfp**#jyD z{Bd_Z3d@9=&Ru5Rdj7NHoLgoQ??d)Rr0?C9w57&mVvX1CixMY)+br4h;r-#?{i|m0 zvx_@*ZwB|bebx8>tL^#n-l{Yy=VnBC{k{K+-0AQZ^6(_1npk>5lfk>V5xaU3Wv8pT*jsz_j zWS?pI_p<|2(n{T)qG{@VQXt{*-PxCP8Qh$3Hk;hvvrR6p@4p3EE>;^+WO-f$A`?Zv z3Lw9IYe8TQl^P(hJFCe4;l01a|5NF*PW$n;HT-~!V*TFyGw03xv-SE*0OGc7O+NVi zo3`&XL1z|VMt7m6#W1{sk_g+LfI8S~ED1q@!e{7pw?|{pd-6)EqtH^zoME5E=M_o^;p{lc#)+cL@ z@$|Tvz&YoycuEfg37mu^P$5OyrfJ$vt=3MvOhSgd z5SPTxY^RJ?{om(=GCJti%jh)Kzl*Wor$62!#OoX#-!LA>%@~Yh|KYf$6U6>-?2m#t z9(DY{AH~BC#K%vq)2WrF5};GhYOd=)=sh9-Pa&SfZ#9!xBG-yAeuNm!1&>D2@9#g1 z_wGL&55o@lN66)X7k>X)LDe!zh9o3IXN7WkgY_awLgJIq`Pa!=j?P~F>+kyS$9Ss# zN6~n%{-Yp@ZT&w7C(a36=1f7x3IsFKT8jm8lvHC1vo%z#8q7smHS!hHIh?O`E;xiF z2#vAP&^baTibAXyUx3jqDNiR(Plm ztc29?zM1sf_lQdO;S;J#aD|SOVg?mm;FJqJ%|y-9F7_torApMblmcbfSoHUj z=s8UfJ9~FFfl{O_Tf<5k4=5EV0j}pZZLR51Os-_tyRIi|?zNjAXp@|Y+-xyt z-F8=%WL$S$#@iJT8SGTssxkTi8fhi8*+3MWnu7iKssy@S?B-fmYT|h*USlClN~9Dw ztSBfEa^WE#e7^8fF<0IfoOvHEFVDRTO?9n2Cexegf!v_6ddE^;%}JU3{P$e!M$uI`|vYE9d>yRrzbTCMQte{Px~ z^XCVlTT_;+=e~ZH8MS==^JaA%sRFYVuWt?N={^4-6c|9@XtnJ223*zwwTp;~fBrc6jBM{T`M XcRo7{#8nV-!wa@K2a@9$G#D5F0;x|D literal 0 HcmV?d00001 diff --git a/test-registry/hello/f.py b/test-registry/hello/f.py deleted file mode 100644 index f048f4749..000000000 --- a/test-registry/hello/f.py +++ /dev/null @@ -1,2 +0,0 @@ -def f(event): - return 'hello' diff --git a/test-registry/hello2.tar.gz b/test-registry/hello2.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f367de7dfe92247fad4f5709ca8d22d419384eb2 GIT binary patch literal 176 zcmb2|=3oE==C>Cexegf!v_6ddE^;%}JU3{P$e!M$uI`|vYE9d>yRrzbTCMQte{Px~ z^XCVlTT_;+=e~ZH8MS==^JaA%sRFYVuWt?N={^4-6c|9@XtnJ223*zwwTp;~fBrc6jBM{T`M XcRo7{#8nV-!wa@K2a@9$G#D5F0;x|D literal 0 HcmV?d00001 diff --git a/test-registry/hello2/f.py b/test-registry/hello2/f.py deleted file mode 100644 index f048f4749..000000000 --- a/test-registry/hello2/f.py +++ /dev/null @@ -1,2 +0,0 @@ -def f(event): - return 'hello' diff --git a/test-registry/install1.tar.gz b/test-registry/install1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..7cdc462c1a67ee246f02170892c262d9a9517da0 GIT binary patch literal 408 zcmV;J0cZXniwFP!000001MSt_Zh|lr2XL-=ij#QZc42KPABhR?u#3F_N*PU{uu^pH z>DSJgICS%3L*sUSH>dwsHC^FaLxtChhLYa<8HNqLL!EqdZ-G4UhKTlZb0UhBkNB(*L zUH3(2x@+G9UB@8*=EeN}8RzmJ9lifU#u?`S5+?ox9!uFlAu9oP%zD#22_ zw6dM8n!0uK8)-^-*qX9YzzO4av8^zrfTF3YW+T-T*xEW%`NDTwTZiih-x^!8657Z@ zrfEO}P7)FXzFRbbwaj4ua`IPYraNI~s?l{;$ycG%l#_taQ^hh@nVSwzP40i#0|{rJ zHkj$7`}Ml2&~zjzOT7N*wOTn~?AhUiboob*{1_oJ4wYnK7~}bFHqvw2lAhDixi>myB}wF-Jc-!p^T^{7^Ai^HD2hGCSseKuMB{JG z^Saig8o=vToig1IaTmt_C_2~TxszH{LYbCIHNqP1fa5s){`}+q`3H&5Jzyj3<;*`n z|FzIY=8~+7RJX#+RIRJDl&?Y)nW>bx>JY7INvF$Fu7cm*FS@Q?|C<+c{WUK2f5O9F z|A#ze*#EaMArp8kWDB`01=wfWnfgg6p;IF=SnZ&Z&1_v)jT_%eQ^3Q{6t#kY^1vqB z2vZ2i>$0r3Qayo)W1&D_AVBGLQ+>!ntiY+YO4BBFY#`R2_3DyV+PSfT4`IpHk|4Al!1bl+Myz$HP zAF_|V>*HQGU5$VGc8U`~zN=CFyLBfz0S8v7Cjck_08FRfiU0rr literal 0 HcmV?d00001 diff --git a/test-registry/install3/f.py b/test-registry/install3/f.py deleted file mode 100644 index 1722c148a..000000000 --- a/test-registry/install3/f.py +++ /dev/null @@ -1,6 +0,0 @@ -import requests -import urllib3 -import simplejson - -def f(event): - return 'imported' diff --git a/test-registry/install3/requirements.in b/test-registry/install3/requirements.in deleted file mode 100644 index 4104bca79..000000000 --- a/test-registry/install3/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -requests -simplejson \ No newline at end of file diff --git a/test-registry/install3/requirements.txt b/test-registry/install3/requirements.txt deleted file mode 100644 index a87b13bd6..000000000 --- a/test-registry/install3/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile requirements.in -# -certifi==2023.7.22 - # via requests -charset-normalizer==3.2.0 - # via requests -idna==3.4 - # via requests -requests==2.31.0 - # via -r requirements.in -simplejson==3.19.1 - # via -r requirements.in -urllib3==2.0.4 - # via requests diff --git a/test-registry/lambda-config-test.tar.gz b/test-registry/lambda-config-test.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..b62f43fb0978db93203ef9a1fe59a2f41f3f2672 GIT binary patch literal 808 zcmV+@1K0c?iwFP!000001MQYuZ`(E$$Nj8NaiBcR25jnX7X}1uL$eDoAi?5d58GhS z(UHVPA~lj~<_7uhBjtL@?5^oUonjq-FElTQ=ZN(A^N?V4azevmk=Zs{WZ~m>&nU{m zcoEK{ESrzQD4b{U2(r_MHX8LxQwcEYH$B(=2i83?{~v_}694F##1^?$gz*#juopa^ zr~iKcNp^7m$s$fh5S}2GBmU?2Up7?Teu$GX|IXgfQ9%XAHw zTU`qd35gP8Y*y$Rp%hIcZW*t@tZgYTmY&_(@Yw#~tuaeB4dXW`^E@JRqd?Z+4Bjvb zgD`p54ZAk-^nyfWaWwf4?RuD;$s<2ZtAg?hrRbG4h~DGsHRG>oY^5(E>CtRUHBy)nSrPDl;k+0Jx$k5K6;{EVs-T%qR`SVBiDTyvq7Lg@S7xIcw7Og)cw9ppQ=W4~tp#EuY&6;m4J8l@C+ zU7M1kK{+-S(}N%;o~>anUS2Gr6$LA|a4XFYM$|wF@B`nLHMT~ve!Z`U%|)+q><6;v zK||Y3K~0ImOIFRa3DN|%6FZgv>Kg$xec!6y(7qa)m*+p7ug?A7C&QuCY&@JpCmGjc zpYdS?Lt>oNnZ&7C;$Mv= literal 0 HcmV?d00001 diff --git a/test-registry/lambda-config-test/f.py b/test-registry/lambda-config-test/f.py deleted file mode 100644 index 9d6cbf34a..000000000 --- a/test-registry/lambda-config-test/f.py +++ /dev/null @@ -1,14 +0,0 @@ -from flask import Flask, request, Response - -def page_not_found(e): - return f"{e}, {request.base_url}, {request.url_root}\n", 404 - -app = Flask("hi") -app.register_error_handler(404, page_not_found) - -# TODO: modify wrappers so "/" is the root -@app.route("/run/lambda-config-test", methods=["GET", "POST", "PUT", "DELETE"]) -def hi(): - print("in hi() of lambda-config-test/f.py") - teapot = 418 # I'm a teapot (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/418) - return Response("hi\n", status=teapot, headers={"A":"B"}) diff --git a/test-registry/lambda-config-test/ol.yaml b/test-registry/lambda-config-test/ol.yaml deleted file mode 100644 index 93d38afe0..000000000 --- a/test-registry/lambda-config-test/ol.yaml +++ /dev/null @@ -1,5 +0,0 @@ -triggers: - http: - - method: GET - - method: POST - diff --git a/test-registry/lambda-config-test/requirements.in b/test-registry/lambda-config-test/requirements.in deleted file mode 100644 index b44f14a28..000000000 --- a/test-registry/lambda-config-test/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -flask==2.3.2 -werkzeug==3.0.3 diff --git a/test-registry/lambda-config-test/requirements.txt b/test-registry/lambda-config-test/requirements.txt deleted file mode 100644 index 96159fc91..000000000 --- a/test-registry/lambda-config-test/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile requirements.in -# -blinker==1.6.2 - # via flask -click==8.1.7 - # via flask -flask==2.3.2 - # via -r requirements.in -itsdangerous==2.1.2 - # via flask -jinja2==3.1.4 - # via flask -markupsafe==2.1.3 - # via - # jinja2 - # werkzeug -werkzeug==3.0.3 - # via - # -r requirements.in - # flask diff --git a/test-registry/max_mem_alloc.tar.gz b/test-registry/max_mem_alloc.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..92af6673e3eec5842a07c4f583b31f9618e4eeea GIT binary patch literal 473 zcmV;~0Ve(*iwFP!000001MSkmirX*{24J846vHk{?2vjT6WLHm54q>oTPZ?`<&D^t z9Aw$qynV-UlA1!>Qz&KUJJ=u%o?jYg&(Ym6RF-0R^HQ+s`zqpse`?@K-6tJ7rH;x{bm0@l|9$l;dI{~!+?uo`MddZRu;d^pD%@ofZaFN=!4JS z*hAw3H0=V~6F!2mI*uQ~>(F@zUwx;canN?Voo?H>(lw|Mv^|8xZqv^(po2FqB(uvp zJXM?Z44we@$8A%=3Oi(V<6o0;O|`A|AsM#5tbnEQdDYZoF|LeX9z242duw(oXfUu@ zLsNs*w(+M0SapJ(SMB~*yzOf2HQUT!4j2%hoaM{;JWp>W7)0uK6OwCJiAN7AbnTe( zs7$~VL$<|`Y$&O}Y2h08@)^)9csumZvJu>qbNqZJdHhRh_SJelulVK<#3V* zDXlV2%7?L`?=ku<1= 0) - - if pid == 0: - # allocate i MB of memory, may crash - buf = 'M' * (mb * 1024**2) - os._exit(0) - - _, status = os.waitpid(pid, 0) - return status == 0 - - -def f(event): - max_attempt = 512 - - for i in range(max_attempt): - if not attempt(i): - return i-1 # we must have succeeded with i-1 MB - - return max_attempt - - -if __name__ == "__main__": - rv = attempt(int(sys.argv[1])) - print(rv) diff --git a/test-registry/numpy21.tar.gz b/test-registry/numpy21.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..b8301e4122b9cf6ebb4b9c0a494699242ed42232 GIT binary patch literal 395 zcmV;60d)Q!iwFP!000001MSt_Zh|lr2k>6=6esaQO=hKhCMLeaE_P3FmeB+XE3J+! zd-oOQqSI_%Y-qNf-%SHOhc+~SpmQ=f*NWq?8(AE>hvPfKLf4Dkh=pN9T;@ihM<5)2 zYlN6arwV|Wm66(fQFmtkm!fmUAB|AFq4mgT}&(ELvN9XUyEGFO%u$L46 z{QRXU>vp;HD08m86?6qh`PWbS=hrxwf6spGKjuGReu(+Mgpo6XyIeGoiGqV2Q=_W~ zF1boIU&E?}TGX@qs;s;Dt}p!*;eSBLsk_ z{cCK={`*5+rz;=9Mx-!PC$-mpQ-2ekXS8mIUE^&2PyYWb2r&N_AsJJ#t!gGJo87tnHLXQ_9q@MJBG*v2XcWtyWq(N`oh0c?UohYZA%qY@2qA6=6esaQO=hKhCMLeaE_P3FmeB+XE3J+! zd-oOQqSI_%Y-qNf-%SHOhc+~SpmQ=f*NWq?8(AE>hvPfKLf4Dkh=pN9T;@ihM<5)2 zYlN6arwV|Wm66(fQFmtkm!fmUAB|AFq4mgT}&(ELvN9XUyEGFO%u$L46 z{QRXU>vp;HsOMaJE9eT2@~@xt&#!SV|DOHWf6RZt{1Eeh2_t6&ce!XF69or5rbbr} zTym9azJ^r`wWw$JRatlQTcLBfZFOEr@G0|cv#roMhpZ}!YAfUe*xE9c>xI)j+B)n% z`q$W!{r88uPFFsHjYwgpPHM0Hrv4^6&uHBayT;l4pZxz>5Mcfr-H1siaoHfrN zoA8az+;q{$QE_dQgy;UEGgiDYg&r?JNImKMXsT4&?%HT$%E@$kvv25YW*e>=QAykP p){`W=J4w=)zF@!+LI@#*5JCtcgb+dqA%qb6u2)}iif#ZX0056l!~g&Q literal 0 HcmV?d00001 diff --git a/test-registry/numpy22/f.py b/test-registry/numpy22/f.py deleted file mode 100644 index 00d4162d2..000000000 --- a/test-registry/numpy22/f.py +++ /dev/null @@ -1,7 +0,0 @@ -import numpy - -def f(event): - return { - 'result': int(numpy.array(event).sum()), - 'numpy-version': numpy.__version__ - } diff --git a/test-registry/numpy22/requirements.in b/test-registry/numpy22/requirements.in deleted file mode 100644 index be533020f..000000000 --- a/test-registry/numpy22/requirements.in +++ /dev/null @@ -1 +0,0 @@ -numpy==2.2 diff --git a/test-registry/numpy22/requirements.txt b/test-registry/numpy22/requirements.txt deleted file mode 100644 index 24937dd6c..000000000 --- a/test-registry/numpy22/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile requirements.in -# -numpy==2.2.0 - # via -r requirements.in diff --git a/test-registry/pandas-tests.tar.gz b/test-registry/pandas-tests.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..b343b475aa012250d4216143d7051c08971521ed GIT binary patch literal 1041 zcmV+s1n&DEiwFP!000001MOB#Z{s!)^;y4yP!6#Ho0h-iMF5}Lo_a`u?kxxiMxrgQ zC6Xnn#7@!w-k~Hrj%7Q0sCU~gp8-qaycrH>I78xL=jw<6^ZCp@NAp>9_}rm0nnmMz z6pv=JxD$<{csA~!+4WuPbV_5XMyOLhmBN-6cpu3BqrgMW{wTR-1ryd_F0L?#OK==d z&*y)3od3yu9Cc811z%eD!Sk=ENT~_b)-q#*Y^$WrnBgX1n}jLLrC4cMs^EM3>2Vz# z)xX`?e}0Y+>VGtu#7Ff%g`cngZ-JZO27S)BK}()91W#I8xnhE8YFUb&wn%X`NMxa$ z{Ds>L{kFB46lj7+5hw!#ItE(GJeMzAtPpSuXexJsGYvS@KRRoGLao(iu^59j90iVl zgPu7>XFJhiF#_u3joQ2+G|(;kJJEr)-B7FJEfz8UG{*6P6DKTk!4oN#d<8z^=$*@r zMNt-N>%1J#@c2;NbXuCM=?Vf>FS@N?^J?>x%g(60T&=3PCZ{~9AB)8tPv418S*mh6 zNFnK^<++>v(@F2Q_7#K?xISr14c}DhiI2TGv>h0!VQWDYE#j!tUEZY zM8>AFwC)ztckfJV_a62t6_^J%gd~@Sb>9s;?ulVwqXm+RiEcQwg&Mw15%$sT%WWTx zj_tG*c7D7m)N5F;9Mxb&ZH8eXf)=b>M-LO?PEoE4S6ONk*iqn0cw&%up{YTO}oLLw!RaB3F( zm4a5-KPzS_ajn@Uig1y#4Jn1<$@+v+!Ht1Tk>>OOMFw$7nMk%Ru50yRk65Wl4w=te z3{77MoaiF8nyZ#{#VpAnln_IA>mV-1Em9@fd6mBw^kW9mP{**=_TtLiatNtrG$jUn z0tGHf^_iIG%Y5fuJ))?#=amf8&rB0o7sPH!L4jnzg;VenHGw82i?3nQJioyH4D8=0 zha+b9^;qxAq)6Rff{ofo;XuNU4-=YXjKJ<*RONt+E!ht_`JizBT_K_T;R@IbA9t28 z+gR8h-G^|nb{qsAq7Z+PTy!6nwaTBaX8!7haDmjwy+bt|@9xLm<6#wrI~y=XuNG)# zAe7*F-K=my8*wAGr51bAp7qOy+n;30@cpm;rr_b>;o;%o;o;%o;o;%o;o;%o@xR00 La~X}`04M+e`qBon literal 0 HcmV?d00001 diff --git a/test-registry/pandas-tests/f.py b/test-registry/pandas-tests/f.py deleted file mode 100644 index a2747e6ec..000000000 --- a/test-registry/pandas-tests/f.py +++ /dev/null @@ -1,42 +0,0 @@ -import numpy -import pandas -import pytest -import inspect -import os, sys -import subprocess -import time - -def f(event): - print("redirecting to stdout.txt, stderr.txt") - sys.stdout.close() - sys.stderr.close() - sys.stdout = open("/tmp/stdout.txt", 'w', 1) - sys.stderr = open("/tmp/stderr.txt", 'w', 1) - - pkg = os.path.dirname(pandas.__file__) - print(pkg) - - # TODO: look into these to determine whether it's a problem - tests_that_fail = [ - "test_oo_optimizable", - "test_oo_optimized_datetime_index_unpickle", - "test_missing_required_dependency", - "test_util_in_top_level", - "test_raw_roundtrip", - "test_get_handle_with_path", - "test_with_missing_lzma", - "test_with_missing_lzma_runtime", - "test_multi_thread_string_io_read_csv", - "test_multi_thread_path_multipart_read_csv", - "test_server_and_default_headers", - "test_server_and_custom_headers", - "test_server_and_all_custom_headers", - "TestS3", - "s3", - ] - - cmd = [pkg, "-o", "cache_dir=/tmp/.my_cache_dir", "-m", "(not slow)"] - cmd.extend(["-k", " and ".join([f"(not {name})" for name in tests_that_fail])]) - print(" ".join(cmd)) - result = pytest.main(cmd) - return result == pytest.ExitCode.OK diff --git a/test-registry/pandas-tests/requirements.in b/test-registry/pandas-tests/requirements.in deleted file mode 100644 index 73c81b29a..000000000 --- a/test-registry/pandas-tests/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -pandas -pytest -hypothesis -exceptiongroup diff --git a/test-registry/pandas-tests/requirements.txt b/test-registry/pandas-tests/requirements.txt deleted file mode 100644 index 9ee986507..000000000 --- a/test-registry/pandas-tests/requirements.txt +++ /dev/null @@ -1,39 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -attrs==23.1.0 - # via hypothesis -exceptiongroup==1.1.3 - # via - # -r requirements.in - # hypothesis - # pytest -hypothesis==6.82.6 - # via -r requirements.in -iniconfig==2.0.0 - # via pytest -numpy==1.25.2 - # via pandas -packaging==23.1 - # via pytest -pandas==2.0.3 - # via -r requirements.in -pluggy==1.3.0 - # via pytest -pytest==7.4.0 - # via -r requirements.in -python-dateutil==2.8.2 - # via pandas -pytz==2023.3 - # via pandas -six==1.16.0 - # via python-dateutil -sortedcontainers==2.4.0 - # via hypothesis -tomli==2.0.1 - # via pytest -tzdata==2023.3 - # via pandas diff --git a/test-registry/pandas-v1.tar.gz b/test-registry/pandas-v1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..c7e4e79e7f83b4472f3522484aa733c4b1de8a0e GIT binary patch literal 474 zcmV<00VVz)iwFP!000001MQVfkDD+MhB@;qM#>?H)HoOis3LKzdhVh3$Xm?D5*TXi z?9!_Ly@nErl5KM-*^N5S#hCd-_B)WRH#>nNNuqA0NfeCNGaikCFbQHBMX?vqAdW&0 zqWN3%Jlhzq0X+L)m9c;EyJEjn$X1K5R%%fSWf~&Y3^lp~j^o{*&!0`tpCvT*Aef<- z3;y%-SGKI%G^Her{hF(cH{B-kU%M~lHca$yp3T3naZUdbqZ9q_D2q`4w_xEf;PX*7 zkjp}VTe&sWqfkO~BQkhsp_cXLOI6mL{Yjc5d}__HQosmhF4%dPqkz0Bit0(KM{vHS z8zg(bOYIoPPyRK|Wi)?C;CKEDI7vP{0c`X|(Ao5BT#cvJIKxYkb>I0WbDw67EYdV2 z?@2g{Ij3yRH)jk&M%bmbk$sm!SS{fh_Av z8|Y=|51Zkh`UFq045@Q<38V({o4v@ljH_`cMlv4vUgK`OJ# za&qE@6%A=@xmvH^!Ll#7ISJj!N;$WFPY#FS&EfFlUKE5N2!bF8f*=TjAP9mW2!bF8 Qf*@~)pYeiIlK?0H03LJdZvX%Q literal 0 HcmV?d00001 diff --git a/test-registry/pandas-v1/f.py b/test-registry/pandas-v1/f.py deleted file mode 100644 index 5e8540a87..000000000 --- a/test-registry/pandas-v1/f.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy -import pandas - -def f(event): - df = pandas.DataFrame(event) - return {'result': int(df.values.sum()), 'numpy-version': numpy.__version__} diff --git a/test-registry/pandas-v1/requirements.in b/test-registry/pandas-v1/requirements.in deleted file mode 100644 index ef6a744f7..000000000 --- a/test-registry/pandas-v1/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -numpy==1.26 -pandas==1.5 diff --git a/test-registry/pandas-v1/requirements.txt b/test-registry/pandas-v1/requirements.txt deleted file mode 100644 index daf20fad5..000000000 --- a/test-registry/pandas-v1/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile requirements.in -# -numpy==1.26.0 - # via - # -r requirements.in - # pandas -pandas==1.5.0 - # via -r requirements.in -python-dateutil==2.8.2 - # via pandas -pytz==2023.3 - # via pandas -six==1.16.0 - # via python-dateutil diff --git a/test-registry/pandas.tar.gz b/test-registry/pandas.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..7a35099c3348aa9a228143cbde57ecd8cc89f875 GIT binary patch literal 473 zcmV;~0Ve(*iwFP!000001MQVfkDD+MhB@;qM#>?H)HpT-P$O}xdhVh3$Xm?D5*X@_ z?6Ot=dkv-1ZrE;mD4VW2&&9yJ;~8oE8nO*$hkCLs=~tE|^zr(OfF(4_Xv&f#4Je~& z5(SXV-&znjW4Q(h+>KM#y`Z}gf1{AC7GIszq87>;BGn8v`T?iu?uGehXY-G<$j_h7 z(94AXeEuz01vlZf+d!_wng8v*{rfd8`9F!{Gym`W4S@cC4Hn@7J|CrlQdR=|%AIX) zg%X-uQNT?Ht!y`6n!4@dchVl=Q)iEjf|#(_CwqlG3MiYZYVM@E1+T6BrLqtEtv!$P z&;B)DqFmi}c^;97#9=RAz)5l#uce>MG`!K1t=gfhP2t;iR#tt^dqPRu7)zhoxU&yl zLZgJRY1GL3JZFSu->`{8^SCFEn6eDH_=qvo|36@x$v>$HMzOD&A6>~arM*;P@J z6R(^w#JGC3UcZCoP;he++Q>$E+3-#dhw;nd@cnK02SE@7K@bE%5ClOG1VIo4K@bE% P{w00@8R58d04M+eYAf&y literal 0 HcmV?d00001 diff --git a/test-registry/pandas/f.py b/test-registry/pandas/f.py deleted file mode 100644 index 5e8540a87..000000000 --- a/test-registry/pandas/f.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy -import pandas - -def f(event): - df = pandas.DataFrame(event) - return {'result': int(df.values.sum()), 'numpy-version': numpy.__version__} diff --git a/test-registry/pandas/requirements.in b/test-registry/pandas/requirements.in deleted file mode 100644 index fb6c7ed7e..000000000 --- a/test-registry/pandas/requirements.in +++ /dev/null @@ -1 +0,0 @@ -pandas diff --git a/test-registry/pandas/requirements.txt b/test-registry/pandas/requirements.txt deleted file mode 100644 index c682c2f24..000000000 --- a/test-registry/pandas/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile requirements.in -# -numpy==2.2.3 - # via pandas -pandas==2.2.3 - # via -r requirements.in -python-dateutil==2.9.0.post0 - # via pandas -pytz==2025.1 - # via pandas -six==1.17.0 - # via python-dateutil -tzdata==2025.1 - # via pandas diff --git a/test-registry/server.tar.gz b/test-registry/server.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..d9cc9530b0a68930078655069401c92476698650 GIT binary patch literal 203 zcmb2|=3oE==C_yqxeqDG94kCuEqZU&>1Af3Rpx&SOFBfooA1?XcPK?zN4nfU?;3DW z-o`FfbyKkI`?sZ6@2z_N)h}apPq<(7>y}HWB9)h%wVx&MQeyhtDN1XDq%Y5|R(@{3 zYR$I9?YF157wJV zlW+QJN4K-Pgr$mJyS?YD{QLiR_J51N{^h?i+wc0@i~d4f3?~0HakDKx>CB+PzyJW& CFJL+V literal 0 HcmV?d00001 diff --git a/test-registry/server/echo.py b/test-registry/server/echo.py deleted file mode 100644 index f209c6344..000000000 --- a/test-registry/server/echo.py +++ /dev/null @@ -1,2 +0,0 @@ -def f(event): - return event diff --git a/test-registry/server/hello.py b/test-registry/server/hello.py deleted file mode 100644 index f048f4749..000000000 --- a/test-registry/server/hello.py +++ /dev/null @@ -1,2 +0,0 @@ -def f(event): - return 'hello' diff --git a/test-registry/syslog.tar.gz b/test-registry/syslog.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..c4e53166e98415ff213bee47ca85679a500f508d GIT binary patch literal 260 zcmb2|=3oE==C>E`XEhs$uzk2==3n^XR9~2Ic9P_qtle*{dtN;fSmX6?%CB$1yADd^ z?4A_(?Qb5RrTPTteJZKl!jl&T6i;mxn*W+zZR!cGg-e5GacXWkC9(2pc(RT~{T3IU zDGNV8Y_HybCO>4-wUY*VzmLgmxb}1V&-6ts{a6!ewupO-tIYxN%{|t`FDmL;Q$D7EHd!Fy-iX9|jEu F1^^fne@g%W literal 0 HcmV?d00001 diff --git a/test-registry/syslog/f.py b/test-registry/syslog/f.py deleted file mode 100644 index 7a9b09869..000000000 --- a/test-registry/syslog/f.py +++ /dev/null @@ -1,6 +0,0 @@ -import subprocess -from ctypes import * - -def f(event): - libc = CDLL("/lib/x86_64-linux-gnu/libc.so.6") - return libc.syscall(103, 0, "") diff --git a/test-registry/thread_counter.tar.gz b/test-registry/thread_counter.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9fdcdd07a29dc8fa614ab07d8f511322a6159cb0 GIT binary patch literal 403 zcmV;E0c`#siwFP!000001MSkgZi6rs24H5M;w%xQjfiOjNR_g5>(VJZBEb+8Fp)8h z^7b_tl3vtyLRHoOTMWL)|FP{Yt;sPhih{27qkz3_N0uM3yNHE;5QG-A~wN!Lt4^jr&W+bPfJRLcXG?)WoX>vc1`}Jv&-i-CFe(% zdHPw+O}fnGFI=c0U0&APq$Jh<6dCDCOevqjJ&8gMr45JTT({EIZTdb}znGk>wV8NL zsJ!kisUh>bXuj!_T(we=^O)=oxzq)v!#32fCwW*ssRIrcH;yCz@UOJCnzYOS`X^X4oIxDatGMPXC<_EIjT9kEj% ze*1q{=7jhO&iH*HlP@=~+Vt3D>ecO^y(Cw6ds!#14Zf9;&aGPZ>*^G*+L%%$t-Z?y zE}MU^J25|G-=Mj`@_$4&(!@+hd3WX R9=Ko5G|w_Zh(Uva0RZ`HY1#k) literal 0 HcmV?d00001 diff --git a/test-registry/timeout/f.py b/test-registry/timeout/f.py deleted file mode 100644 index e77f988a2..000000000 --- a/test-registry/timeout/f.py +++ /dev/null @@ -1,5 +0,0 @@ -import time - -def f(event): - time.sleep(int(event)) - return f"slept {event} second\n" \ No newline at end of file diff --git a/test-registry/version.tar.gz b/test-registry/version.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a2d27d9834554b514b39f39d43acf3187a06765b GIT binary patch literal 325 zcmb2|=3oE==C>Ce^P3DL*dEOLt-bGstA50s)|AfwiPPs!Tk5E>QBowRYv%vDh8Jd& z&k9G1-G3f*t8QujFIH_c%Zr`KH~SNHj$Z%R(;BoUS+F(8&8J0aQnp*suZ*6pz5X(S z+2w1$zSS=^|Mq`n!c_LJg}1DKvAJJ-cKqMzt$SY8@0iHnb7ad#zju57FI%Jie5IvX z&7U>hN1jjBwtPPGoaWuvUwmb^Wc#GfKKf~5aQd9f+w3KKHYXm>yl+*l_n@XOu2N2J zg3#=XmoL6w?d|KlZ_(Ukw!c90fqP%&D-ksRMfp{{>4+kDI;=h+j}kJ z-tx5Tzw)X#)^NtR7k@2Sx@+d|YT4&%>PvHivX*A-UOF=7yw-KpDq9Z literal 0 HcmV?d00001 diff --git a/test-registry/version/f.py b/test-registry/version/f.py deleted file mode 100644 index 8bec67a57..000000000 --- a/test-registry/version/f.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys -import platform - - -def check_versions(): - version = sys.version - return version - -# return Ubuntu and Python versions to be printed to console -def f(event): - #ubuntu_version = platform.linux_distribution()[1] - python_version = sys.version - return python_version - -if __name__ == "__main__": - cv = check_versions() From 4ebf856e563200254101af453e9313df5adfaeed Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Wed, 25 Jun 2025 16:43:19 -0500 Subject: [PATCH 2/7] bugfix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9cb8a5981..49dab43cc 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ wasm-functions: native-functions: imgs/ol-wasm cd bin-functions && cross build --release bash ./bin-functions/install-native.sh test-registry - ls test-registry/hashing.bin test-registry/noop.bin # guarantee they were created + ls test-registry/hashing.tar.gz test-registry/noop.tar.gz # guarantee they were created update-dependencies: cd wasm-image/runtimes/native && cargo update From 89df12025279caec6303da87ed2d5d43299f141f Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Wed, 25 Jun 2025 17:03:03 -0500 Subject: [PATCH 3/7] bugfix --- src/worker/event/sockServer.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/worker/event/sockServer.go b/src/worker/event/sockServer.go index 3a7e9fa4e..f90d67a9b 100644 --- a/src/worker/event/sockServer.go +++ b/src/worker/event/sockServer.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "os/exec" "path/filepath" "strings" "sync" @@ -46,7 +47,28 @@ func (server *SOCKServer) Create(w http.ResponseWriter, _ []string, args map[str } // create args - codeDir := args["code"].(string) + codePath := args["code"].(string) + var codeDir string + + // Handle tar.gz files by extracting them to a temporary directory + if strings.HasSuffix(codePath, ".tar.gz") { + // Create temporary directory for extraction + tempDir, err := ioutil.TempDir("", "ol-sock-code-") + if err != nil { + return fmt.Errorf("failed to create temp dir for tar.gz extraction: %v", err) + } + + // Extract tar.gz file + cmd := exec.Command("tar", "-xzf", codePath, "--directory", tempDir) + if output, err := cmd.CombinedOutput(); err != nil { + os.RemoveAll(tempDir) + return fmt.Errorf("failed to extract tar.gz file %s: %v :: %s", codePath, err, string(output)) + } + + codeDir = tempDir + } else { + codeDir = codePath + } var parent sandbox.Sandbox if p, ok := args["parent"]; ok && p != "" { From e22b601bb8fbc5bf395cedc7aecd1b68ceebd881 Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Wed, 25 Jun 2025 17:07:41 -0500 Subject: [PATCH 4/7] bugfix --- scripts/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.py b/scripts/test.py index ed3ac43ee..d9af48cb2 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -209,7 +209,7 @@ def update_code(): with open(os.path.join(lambda_dir, "f.py"), "w", encoding='utf-8') as code: code.write("def f(event):\n") code.write(f" return {pos}\n") - + # Create tar.gz file tar_path = os.path.join(reg_dir, "version.tar.gz") with tarfile.open(tar_path, "w:gz") as tar: From 7189b3a04c4ae029dbbaa277d5bdf2ad63db368d Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Wed, 25 Jun 2025 17:20:48 -0500 Subject: [PATCH 5/7] bugfix --- scripts/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/test.py b/scripts/test.py index d9af48cb2..f68954b73 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -152,7 +152,6 @@ def call_each_once(lambda_count, alloc_mb=0, zygote_provider="tree"): code.write(" global s\n") code.write(f" s = '*' * {alloc_mb} * 1024**2\n") code.write(f" return {pos}\n") - # Create tar.gz file tar_path = os.path.join(reg_dir, f"L{pos}.tar.gz") with tarfile.open(tar_path, "w:gz") as tar: @@ -209,7 +208,6 @@ def update_code(): with open(os.path.join(lambda_dir, "f.py"), "w", encoding='utf-8') as code: code.write("def f(event):\n") code.write(f" return {pos}\n") - # Create tar.gz file tar_path = os.path.join(reg_dir, "version.tar.gz") with tarfile.open(tar_path, "w:gz") as tar: From b4f7cdd7622e0c986c03dc59dd7bff3145b4d2dc Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Fri, 27 Jun 2025 13:43:34 -0500 Subject: [PATCH 6/7] upload lambda to lambda store on admin install --- src/main.go | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/src/main.go b/src/main.go index 5874a79d3..26fc61563 100644 --- a/src/main.go +++ b/src/main.go @@ -2,6 +2,9 @@ package main import ( + "archive/tar" + "bytes" + "compress/gzip" "encoding/json" "fmt" "io" @@ -13,6 +16,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/open-lambda/open-lambda/ol/bench" "github.com/open-lambda/open-lambda/ol/boss" @@ -300,6 +304,165 @@ func bossStart(ctx *cli.Context) error { return fmt.Errorf("this code should not be reachable") } +// checkBossRunning verifies that the Boss server is running by hitting the /status endpoint +func checkBossRunning() error { + // Load config from boss.json + if err := config.LoadConf("boss.json"); err != nil { + return fmt.Errorf("failed to load boss.json, boss does not seem to be running: %v", err) + } + + // Default values + bossHost := "localhost" + bossPort := config.BossConf.Boss_port + + url := fmt.Sprintf("http://%s:%s/status", bossHost, bossPort) + client := &http.Client{Timeout: 2 * time.Second} + + resp, err := client.Get(url) + if err != nil { + return fmt.Errorf("could not reach boss at %s: %v", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("boss returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// adminInstall corresponds to the "admin install" command +func adminInstall(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return fmt.Errorf("usage: ol admin install ") + } + + if err := checkBossRunning(); err != nil { + return fmt.Errorf("boss is not running or not reachable: %v", err) + } + + funcDir := ctx.Args().Get(0) + funcDir = strings.TrimSuffix(funcDir, "/") + + // Extract function name from directory path + funcName := filepath.Base(funcDir) + + // Check if directory exists + if _, err := os.Stat(funcDir); os.IsNotExist(err) { + return fmt.Errorf("directory %s does not exist", funcDir) + } + + // Create tar.gz archive + tarData, err := createTarGz(funcDir, funcName) + if err != nil { + return fmt.Errorf("failed to create tar.gz: %v", err) + } + + // Upload to lambda store + if err := uploadToLambdaStore(funcName, tarData); err != nil { + return fmt.Errorf("failed to upload to lambda store: %v", err) + } + + fmt.Printf("Successfully installed lambda function: %s\n", funcName) + return nil +} + +// createTarGz creates a tar.gz archive from the function directory +func createTarGz(funcDir, funcName string) ([]byte, error) { + var buf bytes.Buffer + gzWriter := gzip.NewWriter(&buf) + tarWriter := tar.NewWriter(gzWriter) + + // Files to include: f.py (required) and ol.yaml (optional) + filesToInclude := []string{"f.py", "ol.yaml"} + + for _, fileName := range filesToInclude { + filePath := filepath.Join(funcDir, fileName) + + // Check if file exists + info, err := os.Stat(filePath) + if os.IsNotExist(err) { + if fileName == "f.py" { + return nil, fmt.Errorf("required file f.py not found in %s", funcDir) + } + // ol.yaml is optional, skip if not found + continue + } + if err != nil { + return nil, fmt.Errorf("failed to stat %s: %v", filePath, err) + } + + // Create tar header + header := &tar.Header{ + Name: fileName, + Mode: 0644, + Size: info.Size(), + } + + if err := tarWriter.WriteHeader(header); err != nil { + return nil, fmt.Errorf("failed to write tar header for %s: %v", fileName, err) + } + + // Write file content + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %v", filePath, err) + } + + if _, err := io.Copy(tarWriter, file); err != nil { + file.Close() + return nil, fmt.Errorf("failed to write %s to tar: %v", fileName, err) + } + file.Close() + } + + if err := tarWriter.Close(); err != nil { + return nil, fmt.Errorf("failed to close tar writer: %v", err) + } + + if err := gzWriter.Close(); err != nil { + return nil, fmt.Errorf("failed to close gzip writer: %v", err) + } + + return buf.Bytes(), nil +} + +// uploadToLambdaStore uploads the tar.gz data to the lambda store +func uploadToLambdaStore(funcName string, tarData []byte) error { + // Load config from boss.json + if err := config.LoadConf("boss.json"); err != nil { + return fmt.Errorf("failed to load boss.json: %v", err) + } + + // Only works on the same machine as the boss for now + bossHost := "localhost" + bossPort := config.BossConf.Boss_port + + url := fmt.Sprintf("http://%s:%s/registry/%s", bossHost, bossPort, funcName) + + req, err := http.NewRequest("POST", url, bytes.NewReader(tarData)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %v", err) + } + + req.Header.Set("Content-Type", "application/gzip") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + // main runs the admin tool func main() { log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds) @@ -323,6 +486,19 @@ OPTIONS: app.EnableBashCompletion = true app.HideVersion = true app.Commands = []*cli.Command{ + &cli.Command{ + Name: "admin", + Usage: "Admin commands for managing lambdas", + UsageText: "ol admin ", + Subcommands: []*cli.Command{ + { + Name: "install", + Usage: "Install a lambda function from directory", + UsageText: "ol admin install ", + Action: adminInstall, + }, + }, + }, &cli.Command{ Name: "boss", Usage: "Start an OL Boss process", From 2dbc131ff81d4c99f6680592da994ce50eafd5ed Mon Sep 17 00:00:00 2001 From: Mandukhai Alimaa Date: Fri, 27 Jun 2025 14:36:39 -0500 Subject: [PATCH 7/7] upload lambda to lambda store on admin install --- src/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.go b/src/main.go index 26fc61563..e655baaa6 100644 --- a/src/main.go +++ b/src/main.go @@ -354,7 +354,7 @@ func adminInstall(ctx *cli.Context) error { } // Create tar.gz archive - tarData, err := createTarGz(funcDir, funcName) + tarData, err := createTarGz(funcDir) if err != nil { return fmt.Errorf("failed to create tar.gz: %v", err) } @@ -369,7 +369,7 @@ func adminInstall(ctx *cli.Context) error { } // createTarGz creates a tar.gz archive from the function directory -func createTarGz(funcDir, funcName string) ([]byte, error) { +func createTarGz(funcDir string) ([]byte, error) { var buf bytes.Buffer gzWriter := gzip.NewWriter(&buf) tarWriter := tar.NewWriter(gzWriter)