From ded27bcee45adf55f1c97a2f1b36af4c02fa1201 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 7 Apr 2025 12:50:19 +0100 Subject: [PATCH 1/8] Go: Replace `exec.Command("go"` with `toolchain.GoCommand(` --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 6 +++--- go/extractor/toolchain/toolchain.go | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index aeff4f0bd6e1..23e0cb7fe0db 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -67,13 +67,13 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file. func addVersionToMod(version string) bool { - cmd := exec.Command("go", "mod", "edit", "-go="+version) + cmd := toolchain.GoCommand("mod", "edit", "-go="+version) return util.RunCmd(cmd) } // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend func checkVendor() bool { - vendorCheckCmd := exec.Command("go", "list", "-mod=vendor", "./...") + vendorCheckCmd := toolchain.GoCommand("list", "-mod=vendor", "./...") outp, err := vendorCheckCmd.CombinedOutput() if err != nil { badVendorRe := regexp.MustCompile(`(?m)^go: inconsistent vendoring in .*:$`) @@ -438,7 +438,7 @@ func installDependencies(workspace project.GoWorkspace) { util.RunCmd(vendor) } - install = exec.Command("go", "get", "-v", "./...") + install = toolchain.GoCommand("get", "-v", "./...") install.Dir = path log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path) util.RunCmd(install) diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 119e3782f6f6..47df9b11ad09 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -137,9 +137,14 @@ func SupportsWorkspaces() bool { return GetEnvGoSemVer().IsAtLeast(V1_18) } +// Constructs a `*exec.Cmd` for `go` with the specified arguments. +func GoCommand(arg ...string) *exec.Cmd { + return exec.Command("go", arg...) +} + // Run `go mod tidy -e` in the directory given by `path`. func TidyModule(path string) *exec.Cmd { - cmd := exec.Command("go", "mod", "tidy", "-e") + cmd := GoCommand("mod", "tidy", "-e") cmd.Dir = path return cmd } @@ -159,21 +164,21 @@ func InitModule(path string) *exec.Cmd { } } - modInit := exec.Command("go", "mod", "init", moduleName) + modInit := GoCommand("mod", "init", moduleName) modInit.Dir = path return modInit } // Constructs a command to run `go mod vendor -e` in the directory given by `path`. func VendorModule(path string) *exec.Cmd { - modVendor := exec.Command("go", "mod", "vendor", "-e") + modVendor := GoCommand("mod", "vendor", "-e") modVendor.Dir = path return modVendor } // Constructs a command to run `go version`. func Version() *exec.Cmd { - version := exec.Command("go", "version") + version := GoCommand("version") return version } @@ -209,7 +214,7 @@ func RunListWithEnv(format string, patterns []string, additionalEnv []string, fl func ListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) *exec.Cmd { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, patterns...) - cmd := exec.Command("go", args...) + cmd := GoCommand(args...) cmd.Env = append(os.Environ(), additionalEnv...) return cmd } From 0f78e11376315a15a62375e535313851e16cd66e Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Tue, 8 Apr 2025 11:18:40 +0100 Subject: [PATCH 2/8] Go: Detect and apply proxy settings (WIP) --- go/extractor/toolchain/toolchain.go | 4 +- go/extractor/util/BUILD.bazel | 2 + go/extractor/util/registryproxy.go | 114 ++++++++++++++++++++++++ go/extractor/util/registryproxy_test.go | 49 ++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 go/extractor/util/registryproxy.go create mode 100644 go/extractor/util/registryproxy_test.go diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 47df9b11ad09..43581b921005 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -139,7 +139,9 @@ func SupportsWorkspaces() bool { // Constructs a `*exec.Cmd` for `go` with the specified arguments. func GoCommand(arg ...string) *exec.Cmd { - return exec.Command("go", arg...) + cmd := exec.Command("go", arg...) + util.ApplyProxyEnvVars(cmd) + return cmd } // Run `go mod tidy -e` in the directory given by `path`. diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index b7a7783aa799..dd294a63a9c1 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -6,6 +6,7 @@ go_library( name = "util", srcs = [ "extractvendordirs.go", + "registryproxy.go", "semver.go", "util.go", ], @@ -17,6 +18,7 @@ go_library( go_test( name = "util_test", srcs = [ + "registryproxy_test.go", "semver_test.go", "util_test.go", ], diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go new file mode 100644 index 000000000000..8f833c30d90b --- /dev/null +++ b/go/extractor/util/registryproxy.go @@ -0,0 +1,114 @@ +package util + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "strings" +) + +const PROXY_HOST = "CODEQL_PROXY_HOST" +const PROXY_PORT = "CODEQL_PROXY_PORT" +const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE" +const PROXY_URLS = "CODEQL_PROXY_URLS" +const GOPROXY_SERVER = "goproxy_server" + +type RegistryConfig struct { + Type string `json:"type"` + URL string `json:"url"` +} + +var proxy_address string +var proxy_cert_file string +var proxy_configs []RegistryConfig + +// Tries to parse the given string value into an array of RegistryConfig values. +func parseRegistryConfigs(str string) ([]RegistryConfig, error) { + var configs []RegistryConfig + if err := json.Unmarshal([]byte(str), &configs); err != nil { + return nil, err + } + + return configs, nil +} + +func checkEnvVars() { + if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set { + if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set { + proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port) + slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address)) + } + } + + if proxy_cert, proxy_cert_set := os.LookupEnv(PROXY_CA_CERTIFICATE); proxy_cert_set { + // Write the certificate to a temporary file + slog.Info("Found certificate") + + f, err := os.CreateTemp("", "codeql-proxy.crt") + if err != nil { + slog.Error("Unable to create temporary file for the proxy certificate", slog.String("error", err.Error())) + } + + _, err = f.WriteString(proxy_cert) + if err != nil { + slog.Error("Failed to write to the temporary certificate file", slog.String("error", err.Error())) + } + + err = f.Close() + if err != nil { + slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error())) + } else { + proxy_cert_file = f.Name() + } + } + + if proxy_urls, proxy_urls_set := os.LookupEnv(PROXY_URLS); proxy_urls_set { + val, err := parseRegistryConfigs(proxy_urls) + if err != nil { + slog.Error("Unable to parse proxy configurations", slog.String("error", err.Error())) + } else { + // We only care about private registry configurations that are relevant to Go and + // filter others out at this point. + proxy_configs = make([]RegistryConfig, 0) + for _, cfg := range val { + if cfg.Type == GOPROXY_SERVER { + proxy_configs = append(proxy_configs, cfg) + slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) + } + } + } + } +} + +// Applies private package proxy related environment variables to `cmd`. +func ApplyProxyEnvVars(cmd *exec.Cmd) { + slog.Info( + "Applying private registry proxy environment variables", + slog.String("cmd_args", strings.Join(cmd.Args, " ")), + ) + + checkEnvVars() + + if proxy_address != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", proxy_address)) + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + } + if proxy_cert_file != "" { + slog.Info("Setting SSL_CERT_FILE", slog.String("proxy_cert_file", proxy_cert_file)) + cmd.Env = append(cmd.Env, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file)) + } + + if len(proxy_configs) > 0 { + goproxy_val := "https://proxy.golang.org,direct" + + for _, cfg := range proxy_configs { + goproxy_val = cfg.URL + "," + goproxy_val + } + + cmd.Env = append(cmd.Env, fmt.Sprintf("GOPROXY=%s", goproxy_val)) + cmd.Env = append(cmd.Env, "GOPRIVATE=") + cmd.Env = append(cmd.Env, "GONOPROXY=") + } +} diff --git a/go/extractor/util/registryproxy_test.go b/go/extractor/util/registryproxy_test.go new file mode 100644 index 000000000000..b5fde50347f2 --- /dev/null +++ b/go/extractor/util/registryproxy_test.go @@ -0,0 +1,49 @@ +package util + +import ( + "testing" +) + +func parseRegistryConfigsFail(t *testing.T, str string) { + _, err := parseRegistryConfigs(str) + + if err == nil { + t.Fatal("Expected `parseRegistryConfigs` to fail, but it succeeded.") + } +} + +func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig { + val, err := parseRegistryConfigs(str) + + if err != nil { + t.Fatalf("Expected `parseRegistryConfigs` to succeed, but it failed: %s", err.Error()) + } + + return val +} + +func TestParseRegistryConfigs(t *testing.T) { + // parseRegistryConfigsFail(t, "") + + empty := parseRegistryConfigsSuccess(t, "[]") + + if len(empty) != 0 { + t.Fatal("Expected `parseRegistryConfigs(\"[]\")` to return no configurations, but got some.") + } + + single := parseRegistryConfigsSuccess(t, "[{ \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]") + + if len(single) != 1 { + t.Fatalf("Expected `parseRegistryConfigs` to return one configuration, but got %d.", len(single)) + } + + first := single[0] + + if first.Type != "goproxy_server" { + t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", first.Type) + } + + if first.URL != "https://proxy.example.com/mod" { + t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL) + } +} From e210be7bb29b79f1005821ce832dd789ac5fd458 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Tue, 8 Apr 2025 12:38:38 +0100 Subject: [PATCH 3/8] Go: Preserve environment variables in `ApplyProxyEnvVars` --- go/extractor/util/registryproxy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 8f833c30d90b..71071e838146 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -91,6 +91,9 @@ func ApplyProxyEnvVars(cmd *exec.Cmd) { checkEnvVars() + // Preserve environment variables + cmd.Env = os.Environ() + if proxy_address != "" { cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", proxy_address)) cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) From cafe1efefa9c85da7df8d53a61641b15b3fe0ac0 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 25 Apr 2025 12:30:48 +0100 Subject: [PATCH 4/8] Go: Refactor `ApplyProxyEnvVars` --- go/extractor/util/registryproxy.go | 71 +++++++++++++++++++----------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 71071e838146..94c8b7d281ab 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -20,10 +20,23 @@ type RegistryConfig struct { URL string `json:"url"` } +// The address of the proxy including protocol and port (e.g. http://localhost:1234) var proxy_address string + +// The path to the temporary file that stores the proxy certificate, if any. var proxy_cert_file string + +// An array of registry configurations that are relevant to Go. +// This excludes other registry configurations that may be available, but are not relevant to Go. var proxy_configs []RegistryConfig +// Stores the environment variables that we wish to pass on to `go` commands. +var proxy_vars []string = nil + +// Keeps track of whether we have inspected the proxy environment variables. +// Needed since `proxy_vars` may be nil either way. +var proxy_vars_checked bool = false + // Tries to parse the given string value into an array of RegistryConfig values. func parseRegistryConfigs(str string) ([]RegistryConfig, error) { var configs []RegistryConfig @@ -34,10 +47,21 @@ func parseRegistryConfigs(str string) ([]RegistryConfig, error) { return configs, nil } -func checkEnvVars() { +func getEnvVars() []string { + // If `proxy_vars` has been initialised, then we have already performed + // these checks and don't need to do so again. We assume that the environment + // variables are constant throughout the run of the autobuilder. + if proxy_vars != nil { + return proxy_vars + } + + var result []string + if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set { if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set { proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port) + result = append(result, fmt.Sprintf("HTTP_PROXY=%s", proxy_address), fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address)) } } @@ -61,6 +85,7 @@ func checkEnvVars() { slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error())) } else { proxy_cert_file = f.Name() + result = append(result, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file)) } } @@ -71,47 +96,43 @@ func checkEnvVars() { } else { // We only care about private registry configurations that are relevant to Go and // filter others out at this point. - proxy_configs = make([]RegistryConfig, 0) for _, cfg := range val { if cfg.Type == GOPROXY_SERVER { proxy_configs = append(proxy_configs, cfg) slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) } } + + if len(proxy_configs) > 0 { + goproxy_val := "https://proxy.golang.org,direct" + + for _, cfg := range proxy_configs { + goproxy_val = cfg.URL + "," + goproxy_val + } + + result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=") + } } } } // Applies private package proxy related environment variables to `cmd`. func ApplyProxyEnvVars(cmd *exec.Cmd) { - slog.Info( + slog.Debug( "Applying private registry proxy environment variables", slog.String("cmd_args", strings.Join(cmd.Args, " ")), ) - checkEnvVars() - - // Preserve environment variables - cmd.Env = os.Environ() - - if proxy_address != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", proxy_address)) - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + // If we haven't done so yet, check whether the proxy environment variables are set + // and extract information from them. + if !proxy_vars_checked { + proxy_vars = getEnvVars() + proxy_vars_checked = true } - if proxy_cert_file != "" { - slog.Info("Setting SSL_CERT_FILE", slog.String("proxy_cert_file", proxy_cert_file)) - cmd.Env = append(cmd.Env, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file)) - } - - if len(proxy_configs) > 0 { - goproxy_val := "https://proxy.golang.org,direct" - - for _, cfg := range proxy_configs { - goproxy_val = cfg.URL + "," + goproxy_val - } - cmd.Env = append(cmd.Env, fmt.Sprintf("GOPROXY=%s", goproxy_val)) - cmd.Env = append(cmd.Env, "GOPRIVATE=") - cmd.Env = append(cmd.Env, "GONOPROXY=") + // If the proxy is configured, `proxy_vars` will be not `nil`. We append those + // variables Preserve environment variables + if proxy_vars != nil { + cmd.Env = append(os.Environ(), proxy_vars...) } } From 9cfa4514772c252d39b17a28f23dc928672383f7 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 25 Apr 2025 15:41:35 +0100 Subject: [PATCH 5/8] Go: Fix/improve comment about environment variable preservation --- go/extractor/util/registryproxy.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index a0d86bdb0863..85bc03a62a8c 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -133,7 +133,8 @@ func ApplyProxyEnvVars(cmd *exec.Cmd) { } // If the proxy is configured, `proxy_vars` will be not `nil`. We append those - // variables Preserve environment variables + // variables to the existing environment to preserve those environment variables. + // If `cmd.Env` is not changed, then the existing environment is also preserved. if proxy_vars != nil { cmd.Env = append(os.Environ(), proxy_vars...) } From 5172a4d6ec381ae3e6e4453da3b0b2907e83ac2a Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 25 Apr 2025 15:41:57 +0100 Subject: [PATCH 6/8] Go: Remove check from `getEnvVars` --- go/extractor/util/registryproxy.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 85bc03a62a8c..35c592964a37 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -48,13 +48,6 @@ func parseRegistryConfigs(str string) ([]RegistryConfig, error) { } func getEnvVars() []string { - // If `proxy_vars` has been initialised, then we have already performed - // these checks and don't need to do so again. We assume that the environment - // variables are constant throughout the run of the autobuilder. - if proxy_vars != nil { - return proxy_vars - } - var result []string if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set { From 91a794433ae8d5dafb5bab604fdeef61d64e61f0 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 25 Apr 2025 15:42:29 +0100 Subject: [PATCH 7/8] Go: Change "Unable" to "Failed" for consistency --- go/extractor/util/registryproxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 35c592964a37..43eaa461032a 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -65,7 +65,7 @@ func getEnvVars() []string { f, err := os.CreateTemp("", "codeql-proxy.crt") if err != nil { - slog.Error("Unable to create temporary file for the proxy certificate", slog.String("error", err.Error())) + slog.Error("Failed to create temporary file for the proxy certificate", slog.String("error", err.Error())) } _, err = f.WriteString(proxy_cert) From 7592ce47e3cb2ba4ccfb9cc75a975ee6047fb245 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 25 Apr 2025 15:45:12 +0100 Subject: [PATCH 8/8] Go: Restore `parseRegistryConfigsFail` test for the empty string --- go/extractor/util/registryproxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/extractor/util/registryproxy_test.go b/go/extractor/util/registryproxy_test.go index b5fde50347f2..a21b1a215c11 100644 --- a/go/extractor/util/registryproxy_test.go +++ b/go/extractor/util/registryproxy_test.go @@ -23,7 +23,7 @@ func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig { } func TestParseRegistryConfigs(t *testing.T) { - // parseRegistryConfigsFail(t, "") + parseRegistryConfigsFail(t, "") empty := parseRegistryConfigsSuccess(t, "[]")