test(endtoend): Verify all schemas in endtoend

For each test in the endtoend package, use the new sqlc managed databases to verify that the schemas are valid. In the future, we'll attempt to prepare all the queries as well.
sqlc-dev · kyleconroy · Sep 20, 2023 · Sep 20, 2023 · Sep 20, 2023 · Sep 20, 2023
commit 761b1e6b3ba97eb78ecdf1fa6f74d45dc63c4812
diff --git a/internal/endtoend/ddl_test.go b/internal/endtoend/ddl_test.go
@@ -0,0 +1,133 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/sqlc-dev/sqlc/internal/config"
+	"github.com/sqlc-dev/sqlc/internal/migrations"
+	"github.com/sqlc-dev/sqlc/internal/quickdb"
+	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
+	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
+)
+
+func TestValidSchema(t *testing.T) {
+	ctx := context.Background()
+
+	projectID := os.Getenv("DDL_SQLC_PROJECT_ID")
+	authToken := os.Getenv("DDL_SQLC_AUTH_TOKEN")
+
+	if projectID == "" || authToken == "" {
+		if os.Getenv("CI") == "" {
+			t.Skip("skiping ddl tests outside of CI")
+		} else {
+			t.Fatal("missing project id and auth token")
+		}
+	}
+
+	client, err := quickdb.NewClient(projectID, authToken)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	files := []string{}
+
+	// Find all tests that do not have a stderr.txt file
+	err = filepath.Walk("testdata", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Base(path) == "sqlc.json" || filepath.Base(path) == "sqlc.yaml" {
+			stderr := filepath.Join(filepath.Dir(path), "stderr.txt")
+			if _, err := os.Stat(stderr); !os.IsNotExist(err) {
+				return nil
+			}
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, file := range files {
+		file := file // https://golang.org/doc/faq#closures_and_goroutines
+		rd, err := os.Open(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		conf, err := config.ParseConfig(rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for j, pkg := range conf.SQL {
+			j, pkg := j, pkg
+			if pkg.Engine != config.EnginePostgreSQL {
+				continue
+			}
+			t.Run(fmt.Sprintf("endtoend-%s-%d", file, j), func(t *testing.T) {
+				t.Parallel()
+
+				var schema []string
+				for _, path := range pkg.Schema {
+					schema = append(schema, filepath.Join(filepath.Dir(file), path))
+				}
+
+				files, err := sqlpath.Glob(schema)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				var sqls []string
+				for _, f := range files {
+					contents, err := os.ReadFile(f)
+					if err != nil {
+						t.Fatalf("%s: %s", f, err)
+					}
+					// TODO: Split schema into separate files
+					before, _, _ := strings.Cut(string(contents), "-- name:")
+					before, _, _ = strings.Cut(before, "/* name:")
+					// Support loading pg_dump SQL files
+					before = strings.ReplaceAll(before, "CREATE SCHEMA public;", "CREATE SCHEMA IF NOT EXISTS public;")
+					sqls = append(sqls, migrations.RemoveRollbackStatements(before))
+				}
+
+				start := time.Now()
+				resp, err := client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{
+					Engine:     "postgresql",
+					Region:     "sjc",
+					Migrations: sqls,
+				})
+				t.Logf("%s", time.Since(start))
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				t.Cleanup(func() {
+					_, err = client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{
+						DatabaseId: resp.DatabaseId,
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+				})
+
+				conn, err := pgx.Connect(ctx, resp.Uri)
+				if err != nil {
+					t.Fatalf("connect %s: %s", resp.Uri, err)
+				}
+				defer conn.Close(ctx)
+			})
+		}
+	}
+}
diff --git a/internal/quickdb/rpc.go b/internal/quickdb/rpc.go
@@ -0,0 +1,59 @@
+package quickdb
+
+import (
+	"crypto/tls"
+	"os"
+
+	"github.com/riza-io/grpc-go/credentials/basic"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+
+	"github.com/sqlc-dev/sqlc/internal/config"
+	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
+)
+
+const defaultHostname = "grpc.sqlc.dev"
+
+func NewClientFromConfig(cloudConfig config.Cloud) (pb.QuickClient, error) {
+	projectID := cloudConfig.Project
+	authToken := os.Getenv("SQLC_AUTH_TOKEN")
+	return NewClient(projectID, authToken, WithHost(cloudConfig.Hostname))
+}
+
+type options struct {
+	hostname string
+}
+
+type Option func(*options)
+
+func WithHost(host string) Option {
+	return func(o *options) {
+		o.hostname = host
+	}
+}
+
+func NewClient(project, token string, opts ...Option) (pb.QuickClient, error) {
+	var o options
+	for _, apply := range opts {
+		apply(&o)
+	}
+
+	dialOpts := []grpc.DialOption{
+		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
+			InsecureSkipVerify: true,
+		})),
+		grpc.WithPerRPCCredentials(basic.NewPerRPCCredentials(project, token)),
+	}
+
+	hostname := o.hostname
+	if hostname == "" {
+		hostname = defaultHostname
+	}
+
+	conn, err := grpc.Dial(hostname+":443", dialOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return pb.NewQuickClient(conn), nil
+}