8000 [rush-amazon-s3-build-cache-plugin] verify that retry logic works · ricsam/rushstack@b0c5ebf · GitHub
[go: up one dir, main page]
Skip to content

Commit b0c5ebf

Browse files
ricsamricsam
committed
[rush-amazon-s3-build-cache-plugin] verify that retry logic works
1 parent ee86d08 commit b0c5ebf

File tree

7 files changed

+79
-12
lines changed

7 files changed

+79
-12
lines changed

‎build-tests/rush-amazon-s3-build-cache-plugin-integration-test/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
s3data/.minio.sys
2+
s3data/rush-build-cache/test

‎build-tests/rush-amazon-s3-build-cache-plugin-integration-test/README.md

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,32 @@ In this folder run `docker-compose down`
1616
# build the code: rushx build
1717
rushx read-s3-object
1818
```
19+
20+
# Test retries
21+
22+
To test that requests can be retried start the proxy server which will fail every second request:
23+
24+
```bash
25+
rushx start-proxy-server
26+
```
27+
28+
Update the build-cache.json file:
29+
```json
30+
{
31+
"cacheProvider": "amazon-s3",
32+
"amazonS3Configuration": {
33+
"s3Endpoint": "http://localhost:9002",
34+
"s3Region": "us-east-1",
35+
"s3Prefix": "rush-build-cache/test",
36+
"isCacheWriteAllowed": true
37+
}
38+
}
39+
```
40+
41+
Run the rush rebuild command
42+
43+
```bash
44+
cd apps
45+
cd rush
46+
RUSH_BUILD_CACHE_CREDENTIAL="minio:minio123" node lib/start-dev.js --debug rebuild --verbose
47+
```

‎build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
"scripts": {
88
"build": "heft build --clean",
99
"_phase:build": "heft build --clean",
10-
"read-s3-object": "node ./lib/readObject.js"
10+
"read-s3-object": "node ./lib/readObject.js",
11+
"start-proxy-server": "node ./lib/startProxyServer.js"
1112
},
1213
"devDependencies": {
1314
"@rushstack/eslint-config": "workspace:*",
@@ -16,6 +17,8 @@
1617
"@rushstack/node-core-library": "workspace:*",
1718
"@types/node": "12.20.24",
1819
"eslint": "~8.7.0",
19-
"typescript": "~4.5.2"
20+
"typescript": "~4.5.2",
21+
"http-proxy": "~1.18.1",
22+
"@types/http-proxy": "~1.17.8"
2023
}
2124
}

‎build-tests/rush-amazon-s3-build-cache-plugin-integration-test/src/startProxyServer.ts

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import * as httpProxy from 'http-proxy';
2+
import * as http from 'http';
3+
4+
const proxy: httpProxy = httpProxy.createProxyServer({});
5+
6+
const hasFailed: { [k: string]: boolean } = {};
7+
8+
let requestCount: number = 0;
9+
const server: http.Server = http.createServer((req, res) => {
10+
requestCount += 1;
11+
12+
if (req.url && requestCount % 2 === 0 && !hasFailed[req.url]) {
13+
console.log('failing', req.url);
14+
hasFailed[req.url] = true;
15+
res.statusCode = 500;
16+
res.end();
17+
return;
18+
} else if (req.url) {
19+
console.log('proxying', req.url);
20+
}
21+
22+
proxy.web(req, res, {
23+
target: 'http://127.0.0.1:9000'
24+
});
25+
});
26+
27+
server.listen(9002);

‎common/config/rush/nonbrowser-approved-packages.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,10 @@
414414
"name": "html-webpack-plugin",
415415
"allowedCategories": [ "libraries", "tests" ]
416416
},
417+
{
418+
"name": "http-proxy",
419+
"allowedCategories": [ "tests" ]
420+
},
417421
{
418422
"name": "https-proxy-agent",
419423
"allowedCategories": [ "libraries" ]

‎common/config/rush/pnpm-lock.yaml

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rush-plugins/rush-amazon-s3-build-cache-plugin/src/AmazonS3Client.ts

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -402,9 +402,8 @@ export class AmazonS3Client {
402402
}
403403

404404
private async _sendCacheRequestWithRetries<T>(sendRequest: () => Promise<T>): Promise<T> {
405-
const terminal: ITerminal = this._terminal;
406-
407405
type TryResponse = { hasNetworkError: false; response: T } | { hasNetworkError: true; error: unknown };
406+
408407
const trySendRequest: () => Promise<TryResponse> = async (): Promise<TryResponse> => {
409408
try {
410409
const response: T = await sendRequest();
@@ -422,20 +421,20 @@ export class AmazonS3Client {
422421

423422
const response: TryResponse = await trySendRequest();
424423

424+
const log: (...messageParts: (string | IColorableSequence)[]) => void = this._writeDebugLine.bind(this);
425+
425426
if (response.hasNetworkError) {
426427
if (storageRetryOptions && storageRetryOptions.maxTries > 1) {
427-
terminal.writeVerboseLine(
428-
'Network request failed. Will retry request as specified in storageRetryOptions'
429-
);
430-
const { retryDelayInMs, retryPolicyType, maxTries, maxRetryDelayInMs } = storageRetryOptions;
428+
log('Network request failed. Will retry request as specified in storageRetryOptions');
431429
async function retry(retryAttempt: number): Promise<T> {
430+
const { retryDelayInMs, retryPolicyType, maxTries, maxRetryDelayInMs } = storageRetryOptions;
432431
let delay: number = retryDelayInMs;
433432
if (retryPolicyType === StorageRetryPolicyType.EXPONENTIAL) {
434433
delay = retryDelayInMs * Math.pow(2, retryAttempt - 1);
435434
}
436435
delay = Math.min(maxRetryDelayInMs, delay);
437436

438-
terminal.writeVerboseLine(`Will retry request in ${delay}s...`);
437+
log(`Will retry request in ${delay}s...`);
439438
await new Promise<void>((resolve) => {
440439
setTimeout(() => {
441440
resolve();
@@ -446,10 +445,10 @@ export class AmazonS3Client {
446445

447446
if (response.hasNetworkError) {
448447
if (retryAttempt < maxTries - 1) {
449-
terminal.writeVerboseLine('The retried request failed, will try again');
448+
log('The retried request failed, will try again');
450449
return retry(retryAttempt + 1);
451450
} else {
452-
terminal.writeVerboseLine(
451+
log(
453452
'The retried request failed and has reached the maxTries limit, the cloud service is not accessible'
454453
);
455454
throw response.error;
@@ -460,7 +459,7 @@ export class AmazonS3Client {
460459
}
461460
return retry(1);
462461
} else {
463-
terminal.writeVerboseLine('Network request failed and storageRetryOptions is not specified');
462+
log('Network request failed and storageRetryOptions is not specified');
464463
throw response.error;
465464
}
466465
}

0 commit comments

Comments
 (0)
0