8000 Merge pull request #2838 from dmichon-msft/project-cache · microsoft/rushstack@f39d0f7 · GitHub
[go: up one dir, main page]
Skip to content

Commit f39d0f7

Browse files
dmichon-msftdmichon-msft
authored
Merge pull request #2838 from dmichon-msft/project-cache
[rush] Improve startup performance with read cache
2 parents 5e5623c + ff1db15 commit f39d0f7

File tree

7 files changed

+180
-76
lines changed

7 files changed

+180
-76
lines changed

‎apps/rush-lib/src/api/RushProjectConfiguration.ts

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,9 @@ export class RushProjectConfiguration {
126126
}
127127
});
128128

129+
private static readonly _configCache: Map<RushConfigurationProject, RushProjectConfiguration | false> =
130+
new Map();
131+
129132
public readonly project: RushConfigurationProject;
130133

131134
/**
@@ -177,8 +180,17 @@ export class RushProjectConfiguration {
177180
public static async tryLoadForProjectAsync(
178181
project: RushConfigurationProject,
179182
repoCommandLineConfiguration: CommandLineConfiguration | undefined,
180-
terminal: Terminal
183+
terminal: Terminal,
184+
skipCache?: boolean
181185
): Promise<RushProjectConfiguration | undefined> {
186+
// false is a signal that the project config does not exist
187+
const cacheEntry: RushProjectConfiguration | false | undefined = skipCache
188+
? undefined
189+
: RushProjectConfiguration._configCache.get(project);
190+
if (cacheEntry !== undefined) {
191+
return cacheEntry || undefined;
192+
}
193+
182194
const rigConfig: RigConfig = await RigConfig.loadForProjectFolderAsync({
183195
projectFolderPath: project.projectFolder
184196
});
@@ -197,8 +209,11 @@ export class RushProjectConfiguration {
197209
repoCommandLineConfiguration,
198210
terminal
199211
);
200-
return new RushProjectConfiguration(project, rushProjectJson);
212+
const result: RushProjectConfiguration = new RushProjectConfiguration(project, rushProjectJson);
213+
RushProjectConfiguration._configCache.set(project, result);
214+
return result;
201215
} else {
216+
RushProjectConfiguration._configCache.set(project, false);
202217
return undefined;
203218
}
204219
}

‎apps/rush-lib/src/logic/ProjectChangeAnalyzer.ts

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -184,22 +184,21 @@ export class ProjectChangeAnalyzer {
184184
return undefined;
185185
}
186186

187-
const projectHashDeps: Map<string, Map<string, string>> = new Map<string, Map<string, string>>();
187+
const projectHashDeps: Map<string, Map<string, string>> = new Map();
188+
189+
for (const project of this._rushConfiguration.projects) {
190+
projectHashDeps.set(project.packageName, new Map());
191+
}
188192

189193
// Sort each project folder into its own package deps hash
190194
for (const [filePath, fileHash] of repoDeps) {
191195
// findProjectForPosixRelativePath uses LookupByPath, for which lookups are O(K)
192196
// K being the maximum folder depth of any project in rush.json (usually on the order of 3)
193197
const owningProject: RushConfigurationProject | undefined =
194198
this._rushConfiguration.findProjectForPosixRelativePath(filePath);
199+
195200
if (owningProject) {
196-
let owningProjectHashDeps: Map<string, string> | undefined = projectHashDeps.get(
197-
owningProject.packageName
198-
);
199-
if (!owningProjectHashDeps) {
200-
owningProjectHashDeps = new Map<string, string>();
201-
projectHashDeps.set(owningProject.packageName, owningProjectHashDeps);
202-
}
201+
const owningProjectHashDeps: Map<string, string> = projectHashDeps.get(owningProject.packageName)!;
203202
owningProjectHashDeps.set(filePath, fileHash);
204203
}
205204
}
@@ -268,16 +267,15 @@ export class ProjectChangeAnalyzer {
268267
private async _getIgnoreMatcherForProjectAsync(
269268
project: RushConfigurationProject,
270269
terminal: Terminal
271-
): Promise<Ignore> {
270+
): Promise<Ignore | undefined> {
272271
const projectConfiguration: RushProjectConfiguration | undefined =
273272
await RushProjectConfiguration.tryLoadForProjectAsync(project, undefined, terminal);
274-
const ignoreMatcher: Ignore = ignore();
275273

276274
if (projectConfiguration && projectConfiguration.incrementalBuildIgnoredGlobs) {
275+
const ignoreMatcher: Ignore = ignore();
277276
ignoreMatcher.add(projectConfiguration.incrementalBuildIgnoredGlobs);
277+
return ignoreMatcher;
278278
}
279-
280-
return ignoreMatcher;
281279
}
282280

283281
private _getRepoDeps(terminal: Terminal): Map<string, string> | undefined {

‎common/changes/@microsoft/rush/project-cache_2021-08-02-23-04.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@microsoft/rush",
5+
"comment": "Cache rush-project.json reads",
6+
"type": "none"
7+
}
8+
],
9+
"packageName": "@microsoft/rush",
10+
"email": "dmichon-msft@users.noreply.github.com"
11+
}

‎common/changes/@rushstack/rig-package/project-cache_2021-08-02-23-04.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@rushstack/rig-package",
5+
"comment": "Cache rig.json reads",
6+
"type": "minor"
7+
}
8+
],
9+
"packageName": "@rushstack/rig-package",
10+
"email": "dmichon-msft@users.noreply.github.com"
11+
}

‎common/reviews/api/rig-package.api.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
// @public
88
export interface ILoadForProjectFolderOptions {
9+
bypassCache?: boolean;
910
overrideRigJsonObject?: IRigConfigJson;
1011
projectFolderPath: string;
1112
}
@@ -33,7 +34,6 @@ export class RigConfig {
3334
readonly rigProfile: string;
3435
tryResolveConfigFilePath(configFileRelativePath: string): string | undefined;
3536
tryResolveConfigFilePathAsync(configFileRelativePath: string): Promise<string | undefined>;
36-
}
37-
37+
}
3838

3939
```

‎libraries/rig-package/src/RigConfig.ts

Lines changed: 93 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ interface IRigConfigOptions {
4242
rigFound: boolean;
4343
filePath: string;
4444
rigPackageName: string;
45-
rigProfile: string;
45+
rigProfile?: string;
4646
}
4747

4848
/**
@@ -60,6 +60,11 @@ export interface ILoadForProjectFolderOptions {
6060
* If specified, instead of loading the `config/rig.json` from disk, this object will be substituted instead.
6161
*/
6262
overrideRigJsonObject?: IRigConfigJson;
63+
64+
/**
65+
* If specified, force a fresh load instead of returning a cached entry, if one existed.
66+
*/
67+
bypassCache?: boolean;
6368
}
6469

6570
/**
@@ -91,6 +96,8 @@ export class RigConfig {
9196
public static jsonSchemaPath: string = path.resolve(__dirname, './schemas/rig.schema.json');
9297
private static _jsonSchemaObject: object | undefined = undefined;
9398

99+
private static readonly _configCache: Map<string, RigConfig> = new Map();
100+
94101
/**
95102
* The project folder path that was passed to {@link RigConfig.loadForProjectFolder},
96103
* which maybe an absolute or relative path.
@@ -160,13 +167,15 @@ export class RigConfig {
160167
private _resolvedProfileFolder: string | undefined;
161168

162169
private constructor(options: IRigConfigOptions) {
163-
this.projectFolderOriginalPath = options.projectFolderPath;
164-
this.projectFolderPath = path.resolve(options.projectFolderPath);
170+
const { projectFolderPath, rigFound, filePath, rigPackageName, rigProfile = 'default' } = options;
171+
172+
this.projectFolderOriginalPath = projectFolderPath;
173+
this.projectFolderPath = path.resolve( F438 projectFolderPath);
165174

166-
this.rigFound = options.rigFound;
167-
this.filePath = options.filePath;
168-
this.rigPackageName = options.rigPackageName;
169-
this.rigProfile = options.rigProfile;
175+
this.rigFound = rigFound;
176+
this.filePath = filePath;
177+
this.rigPackageName = rigPackageName;
178+
this.rigProfile = rigProfile;
170179

171180
if (this.rigFound) {
172181
this.relativeProfileFolderPath = 'profiles/' + this.rigProfile;
@@ -199,80 +208,110 @@ export class RigConfig {
199208
* equal to `false`.
200209
*/
201210
public static loadForProjectFolder(options: ILoadForProjectFolderOptions): RigConfig {
202-
const rigConfigFilePath: string = path.join(options.projectFolderPath, 'config/rig.json');
211+
const { overrideRigJsonObject, projectFolderPath } = options;
203212

204-
let json: IRigConfigJson;
205-
try {
206-
if (options.overrideRigJsonObject) {
207-
json = options.overrideRigJsonObject;
208-
} else {
209-
if (!fs.existsSync(rigConfigFilePath)) {
210-
return new RigConfig({
211-
projectFolderPath: options.projectFolderPath,
212-
213-
rigFound: false,
214-
filePath: '',
215-
rigPackageName: '',
216-
rigProfile: ''
217-
});
218-
}
213+
const fromCache: RigConfig | undefined =
214+
!options.bypassCache && !overrideRigJsonObject
215+
? RigConfig._configCache.get(projectFolderPath)
216+
: undefined;
219217

218+
if (fromCache) {
219+
return fromCache;
220+
}
221+
222+
const rigConfigFilePath: string = path.join(projectFolderPath, 'config/rig.json');
223+
224+
let config: RigConfig | undefined;
225+
let json: IRigConfigJson | undefined = overrideRigJsonObject;
226+
try {
227+
if (!json) {
220228
const rigConfigFileContent: string = fs.readFileSync(rigConfigFilePath).toString();
221-
json = JSON.parse(stripJsonComments(rigConfigFileContent));
229+
json = JSON.parse(stripJsonComments(rigConfigFileContent)) as IRigConfigJson;
222230
}
223231
RigConfig._validateSchema(json);
224232
} catch (error) {
225-
throw new Error(error.message + '\nError loading config file: ' + rigConfigFilePath);
233+
config = RigConfig._handleConfigError(error, projectFolderPath, rigConfigFilePath);
226234
}
227235

228-
return new RigConfig({
229-
projectFolderPath: options.projectFolderPath,
236+
if (!config) {
237+
config = new RigConfig({
238+
projectFolderPath: projectFolderPath,
230239

231-
rigFound: true,
232-
filePath: rigConfigFilePath,
233-
rigPackageName: json.rigPackageName,
234-
rigProfile: json.rigProfile || 'default'
235-
});
240+
rigFound: true,
241+
filePath: rigConfigFilePath,
242+
rigPackageName: json!.rigPackageName,
243+
rigProfile: json!.rigProfile
244+
});
245+
}
246+
247+
if (!overrideRigJsonObject) {
248+
RigConfig._configCache.set(projectFolderPath, config);
249+
}
250+
return config;
236251
}
237252

238253
/**
239254
* An async variant of {@link RigConfig.loadForProjectFolder}
240255
*/
241256
public static async loadForProjectFolderAsync(options: ILoadForProjectFolderOptions): Promise<RigConfig> {
242-
const rigConfigFilePath: string = path.join(options.projectFolderPath, 'config/rig.json');
257+
const { overrideRigJsonObject, projectFolderPath } = options;
243258

244-
let json: IRigConfigJson;
245-
try {
246-
if (options.overrideRigJsonObject) {
247-
json = options.overrideRigJsonObject;
248-
} else {
249-
if (!(await Helpers.fsExistsAsync(rigConfigFilePath))) {
250-
return new RigConfig({
251-
projectFolderPath: options.projectFolderPath,
252-
253-
rigFound: false,
254-
filePath: '',
255-
rigPackageName: '',
256-
rigProfile: ''
257-
});
258-
}
259+
const fromCache: RigConfig | false | undefined =
260+
!options.bypassCache && !overrideRigJsonObject && RigConfig._configCache.get(projectFolderPath);
261+
262+
if (fromCache) {
263+
return fromCache;
264+
}
265+
266+
const rigConfigFilePath: string = path.join(projectFolderPath, 'config/rig.json');
259267

268+
let config: RigConfig | undefined;
269+
let json: IRigConfigJson | undefined = overrideRigJsonObject;
270+
try {
271+
if (!json) {
260272
const rigConfigFileContent: string = (await fs.promises.readFile(rigConfigFilePath)).toString();
261-
json = JSON.parse(stripJsonComments(rigConfigFileContent));
273+
json = JSON.parse(stripJsonComments(rigConfigFileContent)) as IRigConfigJson;
262274
}
263275

264276
RigConfig._validateSchema(json);
265277
} catch (error) {
278+
config = RigConfig._handleConfigError(error, projectFolderPath, rigConfigFilePath);
279+
}
280+
281+
if (!config) {
282+
config = new RigConfig({
283+
projectFolderPath: projectFolderPath,
284+
285+
rigFound: true,
286+
filePath: rigConfigFilePath,
287+
rigPackageName: json!.rigPackageName,
288+
rigProfile: json!.rigProfile
289+
});
290+
}
291+
292+
if (!overrideRigJsonObject) {
293+
RigConfig._configCache.set(projectFolderPath, config);
294+
}
295+
return config;
296+
}
297+
298+
private static _handleConfigError(
299+
error: NodeJS.ErrnoException,
300+
projectFolderPath: string,
301+
rigConfigFilePath: string
302+
): RigConfig {
303+
if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') {
266304
throw new Error(error.message + '\nError loading config file: ' + rigConfigFilePath);
267305
}
268306

307+
// File not found, i.e. no rig config
269308
return new RigConfig({
270-
projectFolderPath: options.projectFolderPath,
309+
projectFolderPath,
271310

272-
rigFound: true,
273-
filePath: rigConfigFilePath,
274-
rigPackageName: json.rigPackageName,
275-
rigProfile: json.rigProfile || 'default'
311+
rigFound: false,
312+
filePath: '',
313+
rigPackageName: '',
314+
rigProfile: ''
276315
});
277316
}
278317

0 commit comments

Comments
 (0)
0