8000 [rush] feat(buildcache): improve access to operation build cache (#5058) · OlliMartin/rushstack@698fe5c · GitHub < 8000 meta name="release" content="38cb7bfbf9dd21d343725f1f1d774302824cf43d">
[go: up one dir, main page]
Skip to content

Commit 698fe5c

Browse files
aramissennyeydddmichon-msft
aramissennyeydd
and
dmichon-msft
authored
[rush] feat(buildcache): improve access to operation build cache (microsoft#5058)
* feat: improve access to operation build cache Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * add changelog Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * move state hash calculation to OperationExecutionRecord Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * address PR feedback Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * remove unused property Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * remove unused import Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> * Update libraries/rush-lib/src/logic/operations/Operation.ts Co-authored-by: David Michon <dmichon@microsoft.com> * address pr feedback Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> --------- Signed-off-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> Co-authored-by: Aramis Sennyey <aramissennyeydd@users.noreply.github.com> Co-authored-by: David Michon <dmichon@microsoft.com>
1 parent c3f31d2 commit 698fe5c

File tree

4 files changed

+130
-95
lines changed

4 files changed

+130
-95
lines changed

‎common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"comment": "Simplifies the process of going from operation to build cache ID.",
5+
"type": "none",
6+
"packageName": "@microsoft/rush"
7+
}
8+
],
9+
"packageName": "@microsoft/rush",
10+
"email": "aramissennyeydd@users.noreply.github.com"
11+
}

‎libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,33 +13,37 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider';
1313
import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider';
1414
import { TarExecutable } from '../../utilities/TarExecutable';
1515
import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
16+
import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord';
1617

17-
export interface IProjectBuildCacheOptions {
18+
export interface IOperationBuildCacheOptions {
1819
/**
1920
* The repo-wide configuration for the build cache.
2021
*/
2122
buildCacheConfiguration: BuildCacheConfiguration;
2223
/**
23-
* The project to be cached.
24+
* The terminal to use for logging.
2425
*/
25-
project: RushConfigurationProject;
26+
terminal: ITerminal;
27+
}
28+
29+
export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & {
2630
/**
2731
* Value from rush-project.json
2832
*/
2933
projectOutputFolderNames: ReadonlyArray<string>;
3034
/**
31-
* The hash of all relevant inputs and configuration that uniquely identifies this execution.
35+
* The project to be cached.
3236
*/
33-
operationStateHash: string;
37+
project: RushConfigurationProject;
3438
/**
35-
* The terminal to use for logging.
39+
* The hash of all relevant inputs and configuration that uniquely identifies this execution.
3640
*/
37-
terminal: ITerminal;
41+
operationStateHash: string;
3842
/**
3943
* The name of the phase that is being cached.
4044
*/
4145
phaseName: string;
42-
}
46+
};
4347

4448
interface IPathsToCache {
4549
filteredOutputFolderNames: string[];
@@ -94,6 +98,31 @@ export class ProjectBuildCache {
9498
return new ProjectBuildCache(cacheId, options);
9599
}
96100

101+
public static forOperation(
102+
operation: OperationExecutionRecord,
103+
options: IOperationBuildCacheOptions
104+
): ProjectBuildCache {
105+
if (!operation.associatedProject) {
106+
throw new InternalError('Operation must have an associated project');
107+
}
108+
if (!operation.associatedPhase) {
109+
throw new InternalError('Operation must have an associated phase');
110+
}
111+
const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])];
112+
if (operation.metadataFolderPath) {
113+
outputFolders.push(operation.metadataFolderPath);
114+
}
115+
const buildCacheOptions: IProjectBuildCacheOptions = {
116+
...options,
117+
project: operation.associatedProject,
118+
phaseName: operation.associatedPhase.name,
119+
projectOutputFolderNames: outputFolders,
120+
operationStateHash: operation.stateHash
121+
};
122+
const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions);
123+
return new ProjectBuildCache(cacheId, buildCacheOptions);
124+
}
125+
97126
public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean> {
98127
const cacheId: string | undefined = specifiedCacheId || this._cacheId;
99128
if (!cacheId) {

‎libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

Lines changed: 11 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ import type { IPhase } from '../../api/CommandLineConfiguration';
3535
import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
3636
import type { IOperationExecutionResult } from './IOperationExecutionResult';
3737
import type { OperationExecutionRecord } from './OperationExecutionRecord';
38-
import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
3938

4039
const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin';
4140
const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10;
@@ -88,8 +87,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
8887
public apply(hooks: PhasedCommandHooks): void {
8988
const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options;
9089

91-
const { cacheHashSalt } = buildCacheConfiguration;
92-
9390
hooks.beforeExecuteOperations.tap(
9491
PLUGIN_NAME,
9592
(
@@ -104,76 +101,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
104101
);
105102
}
106103

107-
// This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure.
108-
const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot;
109-
110104
const disjointSet: DisjointSet<Operation> | undefined = cobuildConfiguration?.cobuildFeatureEnabled
111105
? new DisjointSet()
112106
: undefined;
113107

114-
const hashByOperation: Map<Operation, string> = new Map();
115-
// Build cache hashes are computed up front to ensure stability and to catch configuration errors early.
116-
function getOrCreateOperationHash(operation: Operation): string {
117-
const cachedHash: string | undefined = hashByOperation.get(operation);
118-
if (cachedHash !== undefined) {
119-
return cachedHash;
120-
}
121-
122-
// Examples of data in the config hash:
123-
// - CLI parameters (ShellOperationRunner)
124-
const configHash: string | undefined = operation.runner?.getConfigHash();
125-
126-
const { associatedProject, associatedPhase } = operation;
127-
// Examples of data in the local state hash:
128-
// - Environment variables specified in `dependsOnEnvVars`
129-
// - Git hashes of tracked files in the associated project
130-
// - Git hash of the shrinkwrap file for the project
131-
// - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
132-
const localStateHash: string | undefined =
133-
associatedProject &&
134-
definitelyDefinedInputsSnapshot.getOperationOwnStateHash(
135-
associatedProject,
136-
associatedPhase?.name
137-
);
138-
139-
// The final state hashes of operation dependencies are factored into the hash to ensure that any
140-
// state changes in dependencies will invalidate the cache.
141-
const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort();
142-
143-
const hasher: crypto.Hash = crypto.createHash('sha1');
144-
// This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
145-
// of the build cache.
146-
hasher.update(`${RushConstants.buildCacheVersion}`);
147-
148-
if (cacheHashSalt !== undefined) {
149-
// This allows repository owners to force a cache bust by changing the salt.
150-
// A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
151-
hasher.update(cacheHashSalt);
152-
}
153-
154-
for (const dependencyHash of dependencyHashes) {
155-
hasher.update(dependencyHash);
156-
}
157-
158-
if (localStateHash) {
159-
hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
160-
}
161-
162-
if (configHash) {
163-
hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
164-
}
165-
166-
const hashString: string = hasher.digest('hex');
167-
168-
hashByOperation.set(operation, hashString);
169-
return hashString;
170-
}
171-
172-
function getDependencyHash(operation: Operation): string {
173-
return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`;
174-
}
175-
176108
for (const [operation, record] of recordByOperation) {
109+
const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({
110+
inputsSnapshot,
111+
buildCacheConfiguration
112+
});
177113
const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation;
178114
if (!associatedProject || !associatedPhase || !runner) {
179115
return;
@@ -188,7 +124,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
188124
// depending on the selected phase.
189125
const fileHashes: ReadonlyMap<string, string> | undefined =
190126
inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName);
191-
const stateHash: string = getOrCreateOperationHash(operation);
192127

193128
const cacheDisabledReason: string | undefined = projectConfiguration
194129
? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp)
@@ -325,10 +260,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
325260
let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({
326261
buildCacheContext,
327262
buildCacheConfiguration,
328-
rushProject: project,
329-
phase,
330263
terminal: buildCacheTerminal,
331-
operation: operation
264+
record
332265
});
333266

334267
// Try to acquire the cobuild lock
@@ -639,39 +572,30 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
639572
private _tryGetProjectBuildCache({
640573
buildCacheConfiguration,
641574
buildCacheContext,
642-
rushProject,
643-
phase,
644575
terminal,
645-
operation
576+
record
646577
}: {
647578
buildCacheContext: IOperationBuildCacheContext;
648579
buildCacheConfiguration: BuildCacheConfiguration | undefined;
649-
rushProject: RushConfigurationProject;
650-
phase: IPhase;
651580
terminal: ITerminal;
652-
operation: Operation;
581+
record: OperationExecutionRecord;
653582
}): ProjectBuildCache | undefined {
654583
if (!buildCacheContext.operationBuildCache) {
655584
const { cacheDisabledReason } = buildCacheContext;
656-
if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) {
585+
if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) {
657586
terminal.writeVerboseLine(cacheDisabledReason);
658587
return;
659588
}
660589

661-
const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext;
662-
if (!outputFolderNames || !buildCacheConfiguration) {
590+
if (!buildCacheConfiguration) {
663591
// Unreachable, since this will have set `cacheDisabledReason`.
664592
return;
665593
}
666594

667595
// eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent
668-
buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({
669-
project: rushProject,
670-
projectOutputFolderNames: outputFolderNames,
596+
buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(record, {
671597
buildCacheConfiguration,
672-
terminal,
673-
operationStateHash,
674-
phaseName: phase.name
598+
terminal
675599
});
676600
}
677601

‎libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
22
// See LICENSE in the project root for license information.
3+
import * as crypto from 'crypto';
34

45
import {
56
type ITerminal,
@@ -29,6 +30,9 @@ import {
2930
initializeProjectLogFilesAsync
3031
} from './ProjectLogWritable';
3132
import type { IOperationExecutionResult } from './IOperationExecutionResult';
33+
import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
34+
import { RushConstants } from '../RushConstants';
35+
import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
3236

3337
export interface IOperationExecutionRecordContext {
3438
streamCollator: StreamCollator;
@@ -114,6 +118,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
114118

115119
private _collatedWriter: CollatedWriter | undefined = undefined;
116120
private _status: OperationStatus;
121+
private _stateHash: string | undefined;
117122

118123
public constructor(operation: Operation, context: IOperationExecutionRecordContext) {
119124
const { runner, associatedPhase, associatedProject } = operation;
@@ -206,6 +211,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
206211
return !this.operation.enabled || this.runner.silent;
207212
}
208213

214+
public get stateHash(): string {
215+
if (!this._stateHash) {
216+
throw new Error(
217+
'Operation state hash is not calculated yet, you must call `calculateStateHash` first.'
218+
);
219+
}
220+
return this._stateHash;
221+
}
222+
209223
/**
210224
* {@inheritdoc IOperationRunnerContext.runWithTerminalAsync}
211225
*/
@@ -335,4 +349,61 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
335349
}
336350
}
337351
}
352+
353+
public calculateStateHash(options: {
354+
inputsSnapshot: IInputsSnapshot;
355+
buildCacheConfiguration: BuildCacheConfiguration;
356+
}): string {
357+
if (!this._stateHash) {
358+
const {
359+
inputsSnapshot,
360+
buildCacheConfiguration: { cacheHashSalt }
361+
} = options;
362+
363+
// Examples of data in the config hash:
364+
// - CLI parameters (ShellOperationRunner)
365+
const configHash: string = this.runner.getConfigHash();
366+
367+
const { associatedProject, associatedPhase } = this;
368+
// Examples of data in the local state hash:
369+
// - Environment variables specified in `dependsOnEnvVars`
370+
// - Git hashes of tracked files in the associated project
371+
// - Git hash of the shrinkwrap file for the project
372+
// - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
373+
const localStateHash: string | undefined =
374+
associatedProject &&
375+
inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name);
376+
377+
// The final state hashes of operation dependencies are factored into the hash to ensure that any
378+
// state changes in dependencies will invalidate the cache.
379+
const dependencyHashes: string[] = Array.from(this.dependencies, (record) => {
380+
return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`;
381+
}).sort();
382+
383+
const hasher: crypto.Hash = crypto.createHash('sha1');
384+
// This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
385+
// of the build cache.
386+
hasher.update(`${RushConstants.buildCacheVersion}`);
387+
388+
if (cacheHashSalt !== undefined) {
389+
// This allows repository owners to force a cache bust by changing the salt.
390+
// A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
391+
hasher.update(cacheHashSalt);
392+
}
393+
394+
for (const dependencyHash of dependencyHashes) {
395+
hasher.update(dependencyHash);
396+
}
397+
398+
if (localStateHash) {
399+
hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
400+
}
401+
402+
hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
403+
404+
const hash: string = hasher.digest('hex');
405+
this._stateHash = hash;
406+
}
407+
return this._stateHash;
408+
}
338409
}

0 commit comments

Comments
 (0)
0