10000 feat: Compress and extract slim binaries with zstd (#2533) · coder/coder@e2785ad · GitHub
[go: up one dir, main page]
Skip to content

Commit e2785ad

Browse files
mafredrideansheather
mafredri
and
deansheather
authored
feat: Compress and extract slim binaries with zstd (#2533)
Fixes #2202 Co-authored-by: Dean Sheather <dean@deansheather.com>
1 parent 64f0473 commit e2785ad

File tree

8 files changed

+377
-16
lines changed

8 files changed

+377
-16
lines changed

‎.github/workflows/coder.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,9 @@ jobs:
370370
- name: Install nfpm
371371
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0
372372

373+
- name: Install zstd
374+
run: sudo apt-get install -y zstd
375+
373376
- name: Build site
374377
run: make -B site/out/index.html
375378

@@ -382,6 +385,7 @@ jobs:
382385
# build slim binaries
383386
./scripts/build_go_slim.sh \
384387
--output ./dist/ \
388+
--compress 22 \
385389
linux:amd64,armv7,arm64 \
386390
windows:amd64,arm64 \
387391
darwin:amd64,arm64

‎.github/workflows/release.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ jobs:
6868
- name: Install nfpm
6969
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0
7070

71+
- name: Install zstd
72+
run: sudo apt-get install -y zstd
73+
7174
- name: Build Site
7275
run: make site/out/index.html
7376

@@ -80,6 +83,7 @@ jobs:
8083
# build slim binaries
8184
./scripts/build_go_slim.sh \
8285
--output ./dist/ \
86+
--compress 22 \
8387
linux:amd64,armv7,arm64 \
8488
windows:amd64,arm64 \
8589
darwin:amd64,arm64
@@ -198,6 +202,9 @@ jobs:
198202
brew tap mitchellh/gon
199203
brew install mitchellh/gon/gon
200204
205+
# Used for compressing embedded slim binaries
206+
brew install zstd
207+
201208
- name: Build Site
202209
run: make site/out/index.html
203210

@@ -210,6 +217,7 @@ jobs:
210217
# build slim binaries
211218
./scripts/build_go_slim.sh \
212219
--output ./dist/ \
220+
--compress 22 \
213221
linux:amd64,armv7,arm64 \
214222
windows:amd64,arm64 \
215223
darwin:amd64,arm64

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ build: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name
3535
# build slim artifacts and copy them to the site output directory
3636
./scripts/build_go_slim.sh \
3737
--version "$(VERSION)" \
38+
--compress 6 \
3839
--output ./dist/ \
3940
linux:amd64,armv7,arm64 \
4041
windows:amd64,arm64 \

‎cli/server.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ func server() *cobra.Command {
254254
Logger: logger.Named("coderd"),
255255
Database: databasefake.New(),
256256
Pubsub: database.NewPubsubInMemory(),
257+
CacheDir: cacheDir,
257258
GoogleTokenValidator: googleTokenValidator,
258259
SecureAuthCookie: secureAuthCookie,
259260
SSHKeygenAlgorithm: sshKeygenAlgorithm,

‎coderd/coderd.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"io"
88
"net/http"
99
"net/url"
10+
"path/filepath"
1011
"sync"
1112
"time"
1213

@@ -42,6 +43,9 @@ type Options struct {
4243
Database database.Store
4344
Pubsub database.Pubsub
4445

46+
// CacheDir is used for caching files served by the API.
47+
CacheDir string
48+
4549
AgentConnectionUpdateFrequency time.Duration
4650
// APIRateLimit is the minutely throughput rate limit per user or ip.
4751
// Setting a rate limit <0 will disable the rate limiter across the entire
@@ -78,11 +82,20 @@ func New(options *Options) *API {
7882
}
7983
}
8084

85+
siteCacheDir := options.CacheDir
86+
if siteCacheDir != "" {
87+
siteCacheDir = filepath.Join(siteCacheDir, "site")
88+
}
89+
binFS, err := site.ExtractOrReadBinFS(siteCacheDir, site.FS())
90+
if err != nil {
91+
panic(xerrors.Errorf("read site bin failed: %w", err))
92+
}
93+
8194
r := chi.NewRouter()
8295
api := &API{
8396
Options: options,
8497
Handler: r,
85-
siteHandler: site.Handler(site.FS()),
98+
siteHandler: site.Handler(site.FS(), binFS),
8699
}
87100
api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgent, 0)
88101

‎scripts/build_go_slim.sh

Lines changed: 27 additions & 2 deletions
< 57AE td data-grid-cell-id="diff-8cc5087dbb31d03c541ddc4760ff3cc89bf6d55b6ac6423ec104d41eff67c2aa-94-117-1" data-selected="false" role="gridcell" style="background-color:var(--diffBlob-additionNum-bgColor, var(--diffBlob-addition-bgColor-num));text-align:center" tabindex="-1" valign="top" class="focusable-grid-cell diff-line-number position-relative left-side">117
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# This script builds multiple "slim" Go binaries for Coder with the given OS and
44
# architecture combinations. This wraps ./build_go_matrix.sh.
55
#
6-
# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] os1:arch1,arch2 os2:arch1 os1:arch3
6+
# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--compress 22] os1:arch1,arch2 os2:arch1 os1:arch3
77
#
88
# If no OS:arch combinations are provided, nothing will happen and no error will
99
# be returned. If no version is specified, defaults to the version from
@@ -15,6 +15,10 @@
1515
#
1616
# The built binaries are additionally copied to the site output directory so
1717
# they can be packaged into non-slim binaries correctly.
18+
#
19+
# When the --compress <level> parameter is provided, the binaries in site/bin
20+
# will be compressed using zstd into site/bin/coder.tar.zst, this helps reduce
21+
# final binary size significantly.
1822

1923
set -euo pipefail
2024
shopt -s nullglob
@@ -23,8 +27,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
2327

2428
version=""
2529
output_path=""
30+
compress=0
2631

27-
args="$(getopt -o "" -l version:,output: -- "$@")"
32+
args="$(getopt -o "" -l version:,output:,compress: -- "$@")"
2833
eval set -- "$args"
2934
while true; do
3035
case "$1" in
@@ -36,6 +41,10 @@ while true; do
3641
output_path="$2"
3742
shift 2
3843
;;
44+
--compress)
45+
compress="$2"
46+
shift 2
47+
;;
3948
--)
4049
shift
4150
break
@@ -48,6 +57,13 @@ done
4857

4958
# Check dependencies
5059
dependencies go
60+
if [[ $compress != 0 ]]; then
61+
dependencies tar zstd
62+
63+
if [[ $compress != [0-9]* ]] || [[ $compress -gt 22 ]] || [[ $compress -lt 1 ]]; then
64+
error "Invalid value for compress, must in in the range of [1, 22]"
65+
fi
66+
fi
5167

5268
# Remove the "v" prefix.
5369
version="${version#v}"
@@ -92,3 +108,12 @@ for f in ./coder-slim_*; do
92108
dest="$dest_dir/$hyphenated"
93109
cp "$f" "$dest"
94110
done
111+
112+
if [[ $compress != 0 ]]; then
113+
log "--- Compressing coder-slim binaries using zstd level $compress ($dest_dir/coder.tar.zst)"
114+
pushd "$dest_dir"
115+
tar cf coder.tar coder-*
116+
rm coder-*
+
zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o coder.tar.zst
118+
popd
119+
fi

‎site/site.go

Lines changed: 151 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
package site
22

33
import (
4+
"archive/tar"
45
"bytes"
56
"context"
6-
7+
"errors"
78
"fmt"
89
"io"
910
"io/fs"
1011
"net/http"
12+
"os"
1113
"path"
1214
"path/filepath"
1315
"strings"
1416
"text/template" // html/template escapes some nonces
1517
"time"
1618

1719
"github.com/justinas/nosurf"
20+
"github.com/klauspost/compress/zstd"
1821
"github.com/unrolled/secure"
22+
"golang.org/x/exp/slices"
1923
"golang.org/x/xerrors"
2024
)
2125

@@ -29,22 +33,25 @@ func WithAPIResponse(ctx context.Context, apiResponse APIResponse) context.Conte
2933
}
3034

3135
// Handler returns an HTTP handler for serving the static site.
32-
func Handler(fileSystem fs.FS) http.Handler {
36+
func Handler(siteFS fs.FS, binFS http.FileSystem) http.Handler {
3337
// html files are handled by a text/template. Non-html files
3438
// are served by the default file server.
3539
//
3640
// REMARK: text/template is needed to inject values on each request like
3741
// CSRF.
38-
files, err := htmlFiles(fileSystem)
39-
42+
files, err := htmlFiles(siteFS)
4043
if err != nil {
4144
panic(xerrors.Errorf("Failed to return handler for static files. Html files failed to load: %w", err))
4245
}
4346

47+
mux := http.NewServeMux()
48+
mux.Handle("/bin/", http.StripPrefix("/bin", http.FileServer(binFS)))
49+
mux.Handle("/", http.FileServer(http.FS(siteFS))) // All other non-html static files.
50+
4451
return secureHeaders(&handler{
45-
fs: fileSystem,
52+
fs: siteFS,
4653
htmlFiles: files,
47-
h: http.FileServer(http.FS(fileSystem)), // All other non-html static files
54+
h: mux,
4855
})
4956
}
5057

@@ -146,8 +153,13 @@ func (h *handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
146153
return
147154
}
148155

156+
switch {
157+
// If requesting binaries, serve straight up.
158+
case reqFile == "bin" || strings.HasPrefix(reqFile, "bin/"):
159+
h.h.ServeHTTP(resp, req)
160+
return
149161
// If the original file path exists we serve it.
150-
if h.exists(reqFile) {
162+
case h.exists(reqFile):
151163
if ShouldCacheFile(reqFile) {
152164
resp.Header().Add("Cache-Control", "public, max-age=31536000, immutable")
153165
}
@@ -357,7 +369,6 @@ func htmlFiles(files fs.FS) (*htmlTemplates, error) {
357369

358370
return nil
359371
})
360-
361372
if err != nil {
362373
return nil, err
363374
}
@@ -366,3 +377,135 @@ func htmlFiles(files fs.FS) (*htmlTemplates, error) {
366377
tpls: root,
367378
}, nil
368379
}
380+
381+
// ExtractOrReadBinFS checks the provided fs for compressed coder
382+
// binaries and extracts them into dest/bin if found. As a fallback,
383+
// the provided FS is checked for a /bin directory, if it is non-empty
384+
// it is returned. Finally dest/bin is returned as a fallback allowing
385+
// binaries to be manually placed in dest (usually
386+
// ${CODER_CACHE_DIRECTORY}/site/bin).
387+
func ExtractOrReadBinFS(dest string, siteFS fs.FS) (http.FileSystem, error) {
388+
if dest == "" {
389+
// No destination on fs, embedded fs is the only option.
390+
binFS, err := fs.Sub(siteFS, "bin")
391+
if err != nil {
392+
return nil, xerrors.Errorf("cache path is empty and embedded fs does not have /bin: %w", err)
393+
}
394+
return http.FS(binFS), nil
395+
}
396+
397+
dest = filepath.Join(dest, "bin")
398+
mkdest := func() (http.FileSystem, error) {
399+
err := os.MkdirAll(dest, 0o700)
400+
if err != nil {
401+
return nil, xerrors.Errorf("mkdir failed: %w", err)
402+
}
403+
return http.Dir(dest), nil
404+
}
405+
406+
archive, err := siteFS.Open("bin/coder.tar.zst")
407+
if err != nil {
408+
if xerrors.Is(err, fs.ErrNotExist) {
409+
files, err := fs.ReadDir(siteFS, "bin")
410+
if err != nil {
411+
if xerrors.Is(err, fs.ErrNotExist) {
412+
// Given fs does not have a bin directory,
413+
// serve from cache directory.
414+
return mkdest()
415+
}
416+
return nil, xerrors.Errorf("site fs read dir failed: %w", err)
417+
}
418+
419+
if len(filterFiles(files, "GITKEEP")) > 0 {
420+
// If there are other files than bin/GITKEEP,
421+
// serve the files.
422+
binFS, err := fs.Sub(siteFS, "bin")
423+
if err != nil {
424+
return nil, xerrors.Errorf("site fs sub dir failed: %w", err)
425+
}
426+
return http.FS(binFS), nil
427+
}
428+
429+
// Nothing we can do, serve the cache directory,
430+
// thus allowing binaries to be places there.
431+
return mkdest()
432+
}
433+
return nil, xerrors.Errorf("open coder binary archive failed: %w", err)
434+
}
435+
defer archive.Close()
436+
437+
dir, err := mkdest()
438+
if err != nil {
439+
return nil, err
440+
}
441+
442+
n, err := extractBin(dest, archive)
443+
if err != nil {
444+
return nil, xerrors.Errorf("extract coder binaries failed: %w", err)
445+
}
446+
if n == 0 {
447+
return nil, xerrors.New("no files were extracted from coder binaries archive")
448+
}
449+
450+
return dir, nil
451+
}
452+
453+
func filterFiles(files []fs.DirEntry, names ...string) []fs.DirEntry {
454+
var filtered []fs.DirEntry
455+
for _, f := range files {
456+
if slices.Contains(names, f.Name()) {
457+
continue
458+
}
459+
filtered = append(filtered, f)
460+
}
461+
return filtered
462+
}
463+
464+
func extractBin(dest string, r io.Reader) (numExtraced int, err error) {
465+
opts := []zstd.DOption{
466+
// Concurrency doesn't help us when decoding the tar and
467+
// can actually slow us down.
468+
zstd.WithDecoderConcurrency(1),
469+
// Ignoring checksums can give a slight performance
470+
// boost but it's probalby not worth the reduced safety.
471+
zstd.IgnoreChecksum(false),
472+
// Allow the decoder to use more memory giving us a 2-3x
473+
// performance boost.
474+
zstd.WithDecoderLowmem(false),
475+
}
476+
zr, err := zstd.NewReader(r, opts...)
477+
if err != nil {
478+
return 0, xerrors.Errorf("open zstd archive failed: %w", err)
479+
}
480+
defer zr.Close()
481+
482+
tr := tar.NewReader(zr)
483+
n := 0
484+
for {
485+
h, err := tr.Next()
486+
if err != nil {
487+
if errors.Is(err, io.EOF) {
488+
return n, nil
489+
}
490+
return n, xerrors.Errorf("read tar archive failed: %w", err)
491+
}
492+
493+
name := filepath.Join(dest, filepath.Base(h.Name))
494+
f, err := os.Create(name)
495+
if err != nil {
496+
return n, xerrors.Errorf("create file failed: %w", err)
497+
}
498+
//#nosec // We created this tar, no risk of decompression bomb.
499+
_, err = io.Copy(f, tr)
500+
if err != nil {
501+
_ = f.Close()
502+
return n, xerrors.Errorf("write file contents failed: %w", err)
503+
}
504+
err = f.Close()
505+
if err != nil {
506+
return n, xerrors.Errorf("close file failed: %w", err)
507+
}
508+
509+
n++
510+
}
511+
}

0 commit comments

Comments
 (0)
0