diff --git a/Dockerfile b/Dockerfile index 564c04c7bb..352c3f9ddd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # STEP 1: Build sqlc -FROM golang:1.24.2 AS builder +FROM golang:1.24.4 AS builder COPY . /workspace WORKDIR /workspace diff --git a/docker-compose.yml b/docker-compose.yml index 1173e8a14a..f318d1ed93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: mysql: - image: "mysql/mysql-server:8.0" + image: "mysql:9" ports: - "3306:3306" restart: always diff --git a/docs/requirements.txt b/docs/requirements.txt index dbca48fde0..3bda4367e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,7 +10,7 @@ docutils==0.20.1 idna==3.10 imagesize==1.4.1 myst-parser==4.0.1 -packaging==24.2 +packaging==25.0 pyparsing==3.2.3 pytz==2025.2 requests==2.32.3 diff --git a/go.mod b/go.mod index 8eca4a1190..3f400daed9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/fatih/structtag v1.2.0 github.com/go-sql-driver/mysql v1.9.2 - github.com/google/cel-go v0.24.1 + github.com/google/cel-go v0.25.0 github.com/google/go-cmp v0.7.0 github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.7.4 @@ -25,14 +25,14 @@ require ( github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.13.0 - google.golang.org/grpc v1.71.1 + google.golang.org/grpc v1.72.0 google.golang.org/protobuf v1.36.6 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.37.0 ) require ( - cel.dev/expr v0.19.1 // indirect + cel.dev/expr v0.23.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect @@ -64,8 +64,8 @@ require ( golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect modernc.org/libc v1.62.1 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index ca9418db2c..64414ebb7d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= -cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= +cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -36,8 +36,8 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.24.1 h1:jsBCtxG8mM5wiUJDSGUqU0K7Mtr3w7Eyv00rw4DiZxI= -github.com/google/cel-go v0.24.1/go.mod h1:Hdf9TqOaTNSFQA1ybQaRqATVoK7m/zcf7IMhGXP5zI8= +github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= +github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -295,12 +295,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= -google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= -google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/internal/codegen/golang/sqlite_type.go b/internal/codegen/golang/sqlite_type.go index 92c13557f6..8a22aaa262 100644 --- a/internal/codegen/golang/sqlite_type.go +++ b/internal/codegen/golang/sqlite_type.go @@ -56,6 +56,9 @@ func sqliteType(req *plugin.GenerateRequest, options *opts.Options, col *plugin. } return "sql.NullTime" + case "json", "jsonb": + return "json.RawMessage" + case "any": return "interface{}" diff --git a/internal/compiler/engine.go b/internal/compiler/engine.go index d263637d9f..f742bfd999 100644 --- a/internal/compiler/engine.go +++ b/internal/compiler/engine.go @@ -23,6 +23,7 @@ type Compiler struct { result *Result analyzer analyzer.Analyzer client dbmanager.Client + selector selector schema []string } @@ -39,12 +40,15 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err case config.EngineSQLite: c.parser = sqlite.NewParser() c.catalog = sqlite.NewCatalog() + c.selector = newSQLiteSelector() case config.EngineMySQL: c.parser = dolphin.NewParser() c.catalog = dolphin.NewCatalog() + c.selector = newDefaultSelector() case config.EnginePostgreSQL: c.parser = postgresql.NewParser() c.catalog = postgresql.NewCatalog() + c.selector = newDefaultSelector() if conf.Database != nil { if conf.Analyzer.Database == nil || *conf.Analyzer.Database { c.analyzer = analyzer.Cached( diff --git a/internal/compiler/expand.go b/internal/compiler/expand.go index 60e654b696..c60b7618b2 100644 --- a/internal/compiler/expand.go +++ b/internal/compiler/expand.go @@ -149,6 +149,11 @@ func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) if counts[cname] > 1 { cname = tableName + "." + cname } + + // This is important for SQLite in particular which needs to + // wrap jsonb column values with `json(colname)` so they're in a + // publicly usable format (i.e. not jsonb). + cname = c.selector.ColumnExpr(cname, column) cols = append(cols, cname) } } diff --git a/internal/compiler/selector.go b/internal/compiler/selector.go new file mode 100644 index 0000000000..04d118ff9c --- /dev/null +++ b/internal/compiler/selector.go @@ -0,0 +1,46 @@ +package compiler + +// selector is an interface used by a compiler for generating expressions for +// output columns in a `SELECT ...` or `RETURNING ...` statement. +// +// This interface is exclusively needed at the moment for SQLite, which must +// wrap output `jsonb` columns with a `json(column_name)` invocation so that a +// publicly consumable format (i.e. not jsonb) is returned. +type selector interface { + // ColumnExpr generates output to be used in a `SELECT ...` or `RETURNING + // ...` statement based on input column name and metadata. + ColumnExpr(name string, column *Column) string +} + +// defaultSelector is a selector implementation that does the simpliest possible +// pass through when generating column expressions. Its use is suitable for all +// database engines not requiring additional customization. +type defaultSelector struct{} + +func newDefaultSelector() *defaultSelector { + return &defaultSelector{} +} + +func (s *defaultSelector) ColumnExpr(name string, column *Column) string { + return name +} + +type sqliteSelector struct{} + +func newSQLiteSelector() *sqliteSelector { + return &sqliteSelector{} +} + +func (s *sqliteSelector) ColumnExpr(name string, column *Column) string { + // Under SQLite, neither json nor jsonb are real data types, and rather just + // of type blob, so database drivers just return whatever raw binary is + // stored as values. This is a problem for jsonb, which is considered an + // internal format to SQLite and no attempt should be made to parse it + // outside of the database itself. For jsonb columns in SQLite, wrap values + // in `json(col)` to coerce the internal binary format to JSON parsable by + // the user-space application. + if column.DataType == "jsonb" { + return "json(" + name + ")" + } + return name +} diff --git a/internal/compiler/selector_test.go b/internal/compiler/selector_test.go new file mode 100644 index 0000000000..e460dd281c --- /dev/null +++ b/internal/compiler/selector_test.go @@ -0,0 +1,35 @@ +package compiler + +import "testing" + +func TestSelector(t *testing.T) { + t.Parallel() + + selectorExpectColumnExpr := func(t *testing.T, selector selector, expected, name string, column *Column) { + if actual := selector.ColumnExpr(name, column); expected != actual { + t.Errorf("Expected %v, got %v for data type %v", expected, actual, column.DataType) + } + } + + t.Run("DefaultSelectorColumnExpr", func(t *testing.T) { + t.Parallel() + + selector := newDefaultSelector() + + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"}) + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"}) + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "jsonb"}) + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"}) + }) + + t.Run("SQLiteSelectorColumnExpr", func(t *testing.T) { + t.Parallel() + + selector := newSQLiteSelector() + + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"}) + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"}) + selectorExpectColumnExpr(t, selector, "json(my_column)", "my_column", &Column{DataType: "jsonb"}) + selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"}) + }) +} diff --git a/internal/endtoend/testdata/jsonb/pgx/go/db.go b/internal/endtoend/testdata/jsonb/pgx/go/db.go new file mode 100644 index 0000000000..e83d6a948c --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/jsonb/pgx/go/models.go b/internal/endtoend/testdata/jsonb/pgx/go/models.go new file mode 100644 index 0000000000..1932ce7f53 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/models.go @@ -0,0 +1,12 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +type Foo struct { + A []byte + B []byte + C []byte + D []byte +} diff --git a/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go b/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go new file mode 100644 index 0000000000..1d7532f6ec --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go @@ -0,0 +1,50 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: query.sql + +package querytest + +import ( + "context" +) + +const insertFoo = `-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + $1, + $2, + $3, + $4 +) RETURNING a, b, c, d +` + +type InsertFooParams struct { + A []byte + B []byte + C []byte + D []byte +} + +func (q *Queries) InsertFoo(ctx context.Context, arg InsertFooParams) error { + _, err := q.db.Exec(ctx, insertFoo, + arg.A, + arg.B, + arg.C, + arg.D, + ) + return err +} + +const selectFoo = `-- name: SelectFoo :exec +SELECT a, b, c, d FROM foo +` + +func (q *Queries) SelectFoo(ctx context.Context) error { + _, err := q.db.Exec(ctx, selectFoo) + return err +} diff --git a/internal/endtoend/testdata/jsonb/pgx/query.sql b/internal/endtoend/testdata/jsonb/pgx/query.sql new file mode 100644 index 0000000000..6959bd1a70 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/query.sql @@ -0,0 +1,15 @@ +-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + @a, + @b, + @c, + @d +) RETURNING *; + +-- name: SelectFoo :exec +SELECT * FROM foo; diff --git a/internal/endtoend/testdata/jsonb/pgx/schema.sql b/internal/endtoend/testdata/jsonb/pgx/schema.sql new file mode 100644 index 0000000000..6b4a1bb0fd --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE foo ( + a json not null, + b jsonb not null, + c json, + d jsonb +); + diff --git a/internal/endtoend/testdata/jsonb/pgx/sqlc.json b/internal/endtoend/testdata/jsonb/pgx/sqlc.json new file mode 100644 index 0000000000..32ede07158 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/sqlc.json @@ -0,0 +1,13 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "engine": "postgresql", + "sql_package": "pgx/v5", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql" + } + ] +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/db.go b/internal/endtoend/testdata/jsonb/sqlite/go/db.go new file mode 100644 index 0000000000..a92cd6e8eb --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/models.go b/internal/endtoend/testdata/jsonb/sqlite/go/models.go new file mode 100644 index 0000000000..7c4f7cd8c7 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/models.go @@ -0,0 +1,16 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +import ( + "encoding/json" +) + +type Foo struct { + A json.RawMessage + B json.RawMessage + C json.RawMessage + D json.RawMessage +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go b/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go new file mode 100644 index 0000000000..9c0858a9c3 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go @@ -0,0 +1,51 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: query.sql + +package querytest + +import ( + "context" + "encoding/json" +) + +const insertFoo = `-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + ?1, + ?2, + ?3, + ?4 +) RETURNING a, json(b), c, json(d) +` + +type InsertFooParams struct { + A json.RawMessage + B json.RawMessage + C json.RawMessage + D json.RawMessage +} + +func (q *Queries) InsertFoo(ctx context.Context, arg InsertFooParams) error { + _, err := q.db.ExecContext(ctx, insertFoo, + arg.A, + arg.B, + arg.C, + arg.D, + ) + return err +} + +const selectFoo = `-- name: SelectFoo :exec +SELECT a, json(b), c, json(d) FROM foo +` + +func (q *Queries) SelectFoo(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, selectFoo) + return err +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/query.sql b/internal/endtoend/testdata/jsonb/sqlite/query.sql new file mode 100644 index 0000000000..6959bd1a70 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/query.sql @@ -0,0 +1,15 @@ +-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + @a, + @b, + @c, + @d +) RETURNING *; + +-- name: SelectFoo :exec +SELECT * FROM foo; diff --git a/internal/endtoend/testdata/jsonb/sqlite/schema.sql b/internal/endtoend/testdata/jsonb/sqlite/schema.sql new file mode 100644 index 0000000000..6b4a1bb0fd --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE foo ( + a json not null, + b jsonb not null, + c json, + d jsonb +); + diff --git a/internal/endtoend/testdata/jsonb/sqlite/sqlc.json b/internal/endtoend/testdata/jsonb/sqlite/sqlc.json new file mode 100644 index 0000000000..cd66df063b --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/sqlc.json @@ -0,0 +1,12 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "engine": "sqlite", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql" + } + ] +}