diff --git a/.gitignore b/.gitignore index 4be2b30ec..61ca68501 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,5 @@ azure.json boss.json cluster.log tasks.log +/venv +.vscode \ No newline at end of file diff --git a/src/common/lambdaConfig.go b/src/common/lambdaConfig.go index fbe286512..d0da7319b 100644 --- a/src/common/lambdaConfig.go +++ b/src/common/lambdaConfig.go @@ -36,8 +36,9 @@ type CronTrigger struct { // TODO: add KafkaTrigger struct // LambdaConfig defines the overall configuration for the lambda function. -type LambdaConfig struct { - Triggers Triggers `yaml:"triggers"` // List of HTTP triggers +type LambdaConfig struct { // List of HTTP triggers + Triggers Triggers `yaml:"triggers"` + MemoryMaxMB *int `yaml:"memory_mb,omitempty"` // Additional configurations can be added here. } diff --git a/src/worker/sandbox/sockPool.go b/src/worker/sandbox/sockPool.go index 28f25a43f..5120a8361 100644 --- a/src/worker/sandbox/sockPool.go +++ b/src/worker/sandbox/sockPool.go @@ -4,11 +4,11 @@ import ( "fmt" "io/ioutil" "log" + "net" + "net/http" "path/filepath" "strings" "sync/atomic" - "net" - "net/http" "time" "github.com/open-lambda/open-lambda/ol/common" @@ -101,7 +101,21 @@ func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir st // don't want to use this cgroup feature, because the child // would take the blame for ALL of the parent's allocations moveMemCharge := (parent == nil) - cSock.cg = pool.cgPool.GetCg(meta.MemLimitMB, moveMemCharge, meta.CPUPercent) + // load lambda-specific config from code directory + lambdaConfig, err := common.LoadLambdaConfig(codeDir) + if err != nil { + // log error and use default settings + pool.printf("could not load lambda config from %s: %v. proceeding with defaults.", codeDir, err) + } + // start with default memory limit + memLimitMB := meta.MemLimitMB + // check if the per-lambda config was loaded AND if the memory field was set + if lambdaConfig != nil && lambdaConfig.MemoryMaxMB != nil { + memLimitMB = *lambdaConfig.MemoryMaxMB + // log memory override + pool.printf("using per-lambda memory limit of %d mb", memLimitMB) + } + cSock.cg = pool.cgPool.GetCg(memLimitMB, moveMemCharge, meta.CPUPercent) t2.T1() cSock.printf("use cgroup %s", cSock.cg.Name()) @@ -192,7 +206,7 @@ func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir st cSock.client = &http.Client{ Transport: &http.Transport{Dial: dial}, - Timeout: time.Second * time.Duration(common.Conf.Limits.Max_runtime_default), + Timeout: time.Second * time.Duration(common.Conf.Limits.Max_runtime_default), } // event handling diff --git a/template.json b/template.json new file mode 100644 index 000000000..19f25517b --- /dev/null +++ b/template.json @@ -0,0 +1,46 @@ +{ + "worker_dir": "", + "worker_url": "localhost", + "worker_port": "5000", + "log_output": true, + "sandbox": "sock", + "server_mode": "lambda", + "registry": "", + "registry_cache_ms": 5000, + "Pkgs_dir": "", + "pip_mirror": "", + "mem_pool_mb": 7337, + "import_cache_tree": "", + "sock_base_path": "", + "sandbox_config": {}, + "docker": { + "runtime": "", + "base_image": "ol-min" + }, + "limits": { + "procs": 10, + "mem_mb": 50, + "cpu_percent": 100, + "max_runtime_default": 30, + "swappiness": 0, + "installer_mem_mb": 500 + }, + "features": { + "reuse_cgroups": false, + "import_cache": "tree", + "downsize_paused_mem": true, + "enable_seccomp": true + }, + "trace": { + "cgroups": false, + "memory": false, + "evictor": false, + "package": false, + "latency": false + }, + "storage": { + "root": "private", + "scratch": "", + "code": "" + } +} \ No newline at end of file diff --git a/worker-1/config.json b/worker-1/config.json new file mode 100644 index 000000000..425b59adc --- /dev/null +++ b/worker-1/config.json @@ -0,0 +1,46 @@ +{ + "worker_dir": "/app/worker-1/worker", + "worker_url": "localhost", + "worker_port": "5000", + "log_output": true, + "sandbox": "sock", + "server_mode": "lambda", + "registry": "/app/worker-1/registry", + "registry_cache_ms": 5000, + "Pkgs_dir": "/app/worker-1/lambda/packages", + "pip_mirror": "", + "mem_pool_mb": 7337, + "import_cache_tree": "/app/worker-1/default-zygotes-40.json", + "sock_base_path": "/app/worker-1/lambda", + "sandbox_config": {}, + "docker": { + "runtime": "", + "base_image": "ol-min" + }, + "limits": { + "procs": 10, + "mem_mb": 50, + "cpu_percent": 100, + "max_runtime_default": 30, + "swappiness": 0, + "installer_mem_mb": 500 + }, + "features": { + "reuse_cgroups": false, + "import_cache": "tree", + "downsize_paused_mem": true, + "enable_seccomp": true + }, + "trace": { + "cgroups": false, + "memory": false, + "evictor": false, + "package": false, + "latency": false + }, + "storage": { + "root": "private", + "scratch": "", + "code": "" + } +} \ No newline at end of file diff --git a/worker-1/default-zygotes-40.json b/worker-1/default-zygotes-40.json new file mode 100644 index 000000000..b492137d9 --- /dev/null +++ b/worker-1/default-zygotes-40.json @@ -0,0 +1,299 @@ +{ + "packages": [], + "split_generation": 0, + "children": [ + { + "packages": [ + "numpy" + ], + "split_generation": 1, + "children": [ + { + "packages": [ + "setuptools" + ], + "split_generation": 3, + "children": [ + { + "packages": [ + "pyparsing" + ], + "split_generation": 4, + "children": [ + { + "packages": [ + "six" + ], + "split_generation": 11, + "children": [ + { + "packages": [ + "python-dateutil" + ], + "split_generation": 13, + "children": [ + { + "packages": [ + "kiwisolver" + ], + "split_generation": 18, + "children": [ + { + "packages": [ + "cycler" + ], + "split_generation": 20, + "children": [ + { + "packages": [ + "matplotlib" + ], + "split_generation": 21, + "children": [ + { + "packages": [ + "pytz" + ], + "split_generation": 31, + "children": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "packages": [ + "six" + ], + "split_generation": 32, + "children": [] + } + ] + }, + { + "packages": [ + "six" + ], + "split_generation": 7, + "children": [ + { + "packages": [ + "python-dateutil" + ], + "split_generation": 8, + "children": [ + { + "packages": [ + "pytz" + ], + "split_generation": 14, + "children": [ + { + "packages": [ + "pandas" + ], + "split_generation": 24, + "children": [] + } + ] + } + ] + }, + { + "packages": [ + "cycler" + ], + "split_generation": 36, + "children": [] + } + ] + }, + { + "packages": [ + "pytz" + ], + "split_generation": 33, + "children": [] + }, + { + "packages": [ + "sqlalchemy" + ], + "split_generation": 37, + "children": [] + } + ] + }, + { + "packages": [ + "six" + ], + "split_generation": 2, + "children": [ + { + "packages": [ + "setuptools" + ], + "split_generation": 5, + "children": [ + { + "packages": [ + "python-dateutil" + ], + "split_generation": 29, + "children": [] + } + ] + }, + { + "packages": [ + "python-dateutil" + ], + "split_generation": 10, + "children": [] + }, + { + "packages": [ + "cycler" + ], + "split_generation": 17, + "children": [ + { + "packages": [ + "urllib3" + ], + "split_generation": 34, + "children": [] + } + ] + }, + { + "packages": [ + "urllib3" + ], + "split_generation": 28, + "children": [] + }, + { + "packages": [ + "mock" + ], + "split_generation": 30, + "children": [] + } + ] + }, + { + "packages": [ + "setuptools" + ], + "split_generation": 6, + "children": [ + { + "packages": [ + "kiwisolver" + ], + "split_generation": 16, + "children": [ + { + "packages": [ + "pytz" + ], + "split_generation": 39, + "children": [] + } + ] + } + ] + }, + { + "packages": [ + "urllib3" + ], + "split_generation": 9, + "children": [ + { + "packages": [ + "chardet" + ], + "split_generation": 25, + "children": [] + } + ] + }, + { + "packages": [ + "pytz" + ], + "split_generation": 12, + "children": [ + { + "packages": [ + "sqlparse" + ], + "split_generation": 26, + "children": [] + } + ] + }, + { + "packages": [ + "pyparsing" + ], + "split_generation": 15, + "children": [ + { + "packages": [ + "sqlalchemy" + ], + "split_generation": 38, + "children": [] + } + ] + }, + { + "packages": [ + "sqlalchemy" + ], + "split_generation": 19, + "children": [] + }, + { + "packages": [ + "werkzeug" + ], + "split_generation": 22, + "children": [ + { + "packages": [ + "click" + ], + "split_generation": 23, + "children": [] + } + ] + }, + { + "packages": [ + "markupsafe" + ], + "split_generation": 27, + "children": [] + }, + { + "packages": [ + "sqlparse" + ], + "split_generation": 35, + "children": [] + } + ] +} \ No newline at end of file diff --git a/worker-1/ol.init b/worker-1/ol.init new file mode 100644 index 000000000..647014543 --- /dev/null +++ b/worker-1/ol.init @@ -0,0 +1 @@ +2025-06-23 14:18:36.624203004 +0000 UTC