8000 chore: retry postgres connection on reset by peer in tests · coder/coder@abc4510 · GitHub
[go: up one dir, main page]

Skip to content

Commit abc4510

Browse files
committed
chore: retry postgres connection on reset by peer in tests
1 parent 73c742a commit abc4510

File tree

1 file changed

+46
-15
lines changed

1 file changed

+46
-15
lines changed

coderd/database/dbtestutil/postgres.go

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@ var (
4545
connectionParamsInitOnce sync.Once
4646
defaultConnectionParams ConnectionParams
4747
errDefaultConnectionParamsInit error
48+
retryableErrSubstrings = []string{
49+
"connection reset by peer",
50+
}
51+
noPostgresRunningErrSubstrings = []string{
52+
"connection refused", // nothing is listening on the port
53+
"No connection could be made", // Windows variant of the above
54+
}
4855
)
4956

5057
// initDefaultConnection initializes the default postgres connection parameters.
@@ -59,28 +66,42 @@ func initDefaultConnection(t TBSubset) error {
5966
DBName: "postgres",
6067
}
6168
dsn := params.DSN()
62-
db, dbErr := sql.Open("postgres", dsn)
63-
if dbErr == nil {
64-
dbErr = db.Ping()
65-
if closeErr := db.Close(); closeErr != nil {
66-
return xerrors.Errorf("close db: %w", closeErr)
69+
70+
// Helper closure to try opening and pinging the default Postgres instance.
71+
// Used within a single retry loop that handles both retryable and permanent errors.
72+
attemptConn := func() error {
73+
db, err := sql.Open("postgres", dsn)
74+
if err == nil {
75+
err = db.Ping()
76+
if closeErr := db.Close(); closeErr != nil {
77+
return xerrors.Errorf("close db: %w", closeErr)
78+
}
6779
}
80+
return err
6881
}
82+
83+
var dbErr error
6984
shouldOpenContainer := false
70-
if dbErr != nil {
71-
errSubstrings := []string{
72-
"connection refused", // this happens on Linux when there's nothing listening on the port
73-
"No connection could be made", // like above but Windows
85+
// Retry up to 3 seconds for temporary errors.
86+
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
87+
defer cancel()
88+
for r := retry.New(10*time.Millisecond, 500*time.Millisecond); r.Wait(ctx); {
89+
dbErr = attemptConn()
90+
if dbErr == nil {
91+
break
7492
}
7593
errString := dbErr.Error()
76-
for _, errSubstring := range errSubstrings {
77-
if strings.Contains(errString, errSubstring) {
78-
shouldOpenContainer = true
79-
break
80-
}
94+
if !containsAnySubstring(errString, retryableErrSubstrings) {
95+
break
8196
}
97+
t.Logf("failed to connect to postgres, retrying: %s", errString)
98+
}
99+
100+
// After the loop dbErr is the last connection error (if any).
101+
if dbErr != nil && containsAnySubstring(dbErr.Error(), noPostgresRunningErrSubstrings) {
102+
shouldOpenContainer = true
82103
}
83-
if dbErr != nil && shouldOpenContainer {
104+
if shouldOpenContainer {
84105
// If there's no database running on the default port, we'll start a
85106
// postgres container. We won't be cleaning it up so it can be reused
86107
// by subsequent tests. It'll keep on running until the user terminates
@@ -110,6 +131,7 @@ func initDefaultConnection(t TBSubset) error {
110131
if connErr == nil {
111132
break
112133
}
134+
t.Logf("failed to connect to postgres after starting container, may retry: %s", connErr.Error())
113135
}
114136
} else if dbErr != nil {
115137
return xerrors.Errorf("open postgres connection: %w", dbErr)
@@ -523,3 +545,12 @@ func OpenContainerized(t TBSubset, opts DBContainerOptions) (string, func(), err
523545

524546
return dbURL, containerCleanup, nil
525547
}
548+
549+
func containsAnySubstring(s string, substrings []string) bool {
550+
for _, substr := range substrings {
551+
if strings.Contains(s, substr) {
552+
return true
553+
}
554+
}
555+
return false
556+
}

0 commit comments

Comments
 (0)
0