10000 fix: improve progress reporting (#48) · sourcegraph/scip-python@07ddc39 · GitHub
[go: up one dir, main page]
Skip to content

Commit 07ddc39

Browse files
tjdevriestjdevries
authored
fix: improve progress reporting (#48)
* remove spinner * add much better progress (without flooding logs). Need to add config still * wire up to command line args * remove temp debug
1 parent b4e352a commit 07ddc39

File tree

9 files changed

+273
-70
lines changed

9 files changed

+273
-70
lines changed

‎packages/pyright-scip/package-lock.json

Lines changed: 84 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/pyright-scip/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"@types/google-protobuf": "^3.15.5",
3030
"@types/jest": "^27.5.0",
3131
"@types/ora": "^3.2.0",
32+
"clean-terminal-webpack-plugin": "^3.0.0",
3233
"copy-webpack-plugin": "^10.2.0",
3334
"jest": "^27.4.7",
3435
"jest-junit": "^13.0.0",
@@ -45,7 +46,6 @@
4546
"diff": "^5.0.0",
4647
"glob": "^7.2.0",
4748
"google-protobuf": "^3.19.3",
48-
"ora": "^6.1.0",
4949
"ts-node": "^10.5.0",
5050
"vscode-languageserver": "^7.0.0"
5151
},

‎packages/pyright-scip/src/MainCommand.test.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@ function checkIndexParser(args: string[], expectedOptions: Partial<IndexOptions>
1818

1919
// defaults
2020
checkIndexParser([], {
21-
progressBar: true,
21+
quiet: false,
2222
cwd: process.cwd(),
2323
output: DEFAULT_OUTPUT_FILE,
2424
projectName: 'snapshot-util',
2525
});
2626

2727
checkIndexParser(['--cwd', 'qux'], { cwd: 'qux' });
28-
checkIndexParser(['--no-progress-bar'], { progressBar: false });
28+
checkIndexParser(['--no-progress-bar'], { quiet: false });
29+
checkIndexParser(['--show-progress-rate-limit', '120'], { showProgressRateLimit: 120 });
30+
checkIndexParser(['--show-progress-rate-limit', '0.5'], { showProgressRateLimit: 0.5 });

‎packages/pyright-scip/src/MainCommand.ts

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Command } from 'commander';
1+
import { Command, InvalidArgumentError } from 'commander';
22
import packageJson from '../package.json';
33

44
export interface IndexOptions {
@@ -12,7 +12,10 @@ export interface IndexOptions {
1212
exclude: string;
1313
output: string;
1414
cwd: string;
15-
progressBar: boolean;
15+
16+
// Progress reporting configuration
17+
quiet: boolean;
18+
showProgressRateLimit: number | undefined;
1619
}
1720

1821
export interface SnapshotOptions extends IndexOptions {
@@ -27,6 +30,19 @@ export interface EnvironmentOptions {
2730

2831
export const DEFAULT_OUTPUT_FILE = 'index.scip';
2932

33+
const parseOptionalNum = (value: string) => {
34+
if (value === undefined) {
35+
return undefined;
36+
}
37+
38+
// parseInt takes a string and a radix
39+
const parsedValue = parseFloat(value);
40+
if (isNaN(parsedValue)) {
41+
throw new InvalidArgumentError('Not a number.');
42+
}
43+
return parsedValue;
44+
};
45+
3046
export function mainCommand(
3147
indexAction: (options: IndexOptions) => void,
3248
snapshotAction: (dir: string, options: SnapshotOptions) => void,
@@ -42,7 +58,13 @@ export function mainCommand(
4258
.option('--cwd <path>', 'working directory for executing scip-python', process.cwd())
4359
.option('--output <path>', 'path to the output file', DEFAULT_OUTPUT_FILE)
4460
.option('--snapshot-dir <path>', 'the directory to output a snapshot of the SCIP dump')
45-
.option('--no-progress-bar', 'whether to disable the progress bar')
61+
.option('--no-progress-bar', '(deprecated, use "--quiet")')
62+
.option('--quiet', 'run without logging and status information', false)
63+
.option(
64+
'--show-progress-rate-limit <limit>',
65+
'minimum number of seconds between progress messages in the output.',
66+
parseOptionalNum
67+
)
4668
.option('--environment <json-file>', 'the environment json file (experimental)')
4769
.option('--include <pattern>', 'comma-separated list of patterns to include (experimental)')
4870
.option('--exclude <pattern>', 'comma-separated list of patterns to exclude (experimental)')
@@ -60,8 +82,14 @@ export function mainCommand(
6082
.option('--project-version <version>', 'the name of the current project, pypi name if applicable', '0.1')
6183
.option('--output <path>', 'path to the output file', DEFAULT_OUTPUT_FILE)
6284
.option('--environment <json-file>', 'the environment json file (experimental)')
63-
.option('--no-index', 'skip indexing (and just use existing index.scip)')
64-
.option('--no-progress-bar', 'whether to disable the progress bar')
85+
.option('--no-index', 'skip indexing (use existing index.scip)')
86+
.option('--no-progress-bar', '(deprecated, use "--quiet")')
87+
.option('--quiet', 'run without logging and status information', false)
88+
.option(
89+
'--show-progress-rate-limit <limit>',
90+
'minimum number of seconds between progress messages in the output.',
91+
parseOptionalNum
92+
)
6593
.action((dir, parsedOptions) => {
6694
snapshotAction(dir, parsedOptions as SnapshotOptions);
6795
});

‎packages/pyright-scip/src/indexer.ts

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import getEnvironment from './virtualenv/environment';
1818
import { version } from 'package.json';
1919
import { getFileSpec } from 'pyright-internal/common/pathUtils';
2020
import { FileMatcher } from './FileMatcher';
21-
import { withStatus } from './status';
21+
import { sendStatus, StatusUpdater, withStatus } from './status';
2222
import { scip } from './scip';
2323

2424
export class Indexer {
@@ -77,6 +77,14 @@ export class Indexer {
7777
onCancellationRequested: Event.None,
7878
};
7979

80+
F987 const analyzer_fn = (progress: StatusUpdater) => {
81+
while (this.program.analyze({ openFilesTimeInMs: 10000, noOpenFilesTimeInMs: 10000 })) {
82+
const filesCompleted = this.program.getFileCount() - this.program.getFilesToAnalyzeCount();
83+
const filesTotal = this.program.getFileCount();
84+
progress.message(`${filesCompleted} / ${filesTotal}`);
85+
}
86+
};
87+
8088
const packageConfig = getEnvironment(
8189
this.projectFiles,
8290
this.scipConfig.projectVersion,
@@ -101,15 +109,12 @@ export class Indexer {
101109
);
102110

103111
// Run program analysis once.
104-
withStatus('Parse and search for dependencies', () => {
105-
while (this.program.analyze()) {}
106-
});
112+
withStatus('Parse and search for dependencies', analyzer_fn);
107113

114+
// let projectSourceFiles: SourceFile[] = this.program.getTracked().map((f) => f.sourceFile);
108115
let projectSourceFiles: SourceFile[] = [];
109-
withStatus('Index workspace and track project files', (spinner) => {
110-
this.program.indexWorkspace((filepath: string, _results: IndexResults) => {
111-
spinner.render();
112-
116+
withStatus('Index workspace and track project files', () => {
117+
this.program.indexWorkspace((filepath: string) => {
113118
// Filter out filepaths not part of this project
114119
if (filepath.indexOf(this.scipConfig.projectRoot) != 0) {
115120
return;
@@ -128,18 +133,18 @@ export class Indexer {
128133
});
129134

130135
// Mark every original sourceFile as dirty so that we can
131-
// visit them via the program again (with all dependencies noted)
136+
// visit them and only them via the program again (with all dependencies noted)
132137
projectSourceFiles.forEach((sourceFile) => {
133138
sourceFile.markDirty(true);
134139
});
135140

136-
while (this.program.analyze()) {}
141+
withStatus('Analyze project and dependencies', analyzer_fn);
137142

138143
let externalSymbols: Map<string, scip.SymbolInformation> = new Map();
139-
withStatus('Parse and emit SCIP', (spinner) => {
144+
withStatus('Parse and emit SCIP', (progress) => {
140145
const typeEvaluator = this.program.evaluator!;
141-
projectSourceFiles.forEach((sourceFile) => {
142-
spinner.render();
146+
projectSourceFiles.forEach((sourceFile, index) => {
147+
progress.progress(`(${index}/${projectSourceFiles.length}): ${sourceFile.getFilePath()}`);
143148

144149
const filepath = sourceFile.getFilePath();
145150
let doc = new scip.Document({
@@ -160,7 +165,15 @@ export class Indexer {
160165
pythonEnvironment: packageConfig,
161166
globalSymbols,
162167
});
163-
visitor.walk(tree);
168+
169+
try {
170+
visitor.walk(tree);
171+
} catch (e) {
172+
throw {
173+
currentFilepath: sourceFile.getFilePath(),
174+
error: e,
175+
};
176+
}
164177

165178
if (doc.occurrences.length === 0) {
166179
console.log(`file:${filepath} had no occurrences`);
@@ -173,10 +186,14 @@ export class Indexer {
173186
})
174187
);
175188
});
189+
});
176190

191+
withStatus('Writing external symbols to SCIP index', () => {
177192
const externalSymbolIndex = new scip.Index();
178193
externalSymbolIndex.external_symbols = Array.from(externalSymbols.values());
179194
this.scipConfig.writeIndex(externalSymbolIndex);
180195
});
196+
197+
sendStatus(`Sucessfully wrote SCIP index to ${this.scipConfig.output}`);
181198
}
182199
}

‎packages/pyright-scip/src/main.ts

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,17 @@ import { diffSnapshot, formatSnapshot, writeSnapshot } from './lib';
77
import { Input } from './lsif-typescript/Input';
88
import { join } from 'path';
99
import { mainCommand } from './MainCommand';
10-
import { withStatus, statusConfig } from './status';
10+
import { sendStatus, setQuiet, setShowProgressRateLimit } from './status';
1111
import { Indexer } from './indexer';
1212
import { exit } from 'process';
1313

1414
export function main(): void {
1515
const command = mainCommand(
1616
(options) => {
17-
if (!options.progressBar) {
18-
statusConfig.showProgress = false;
17+
setQuiet(options.quiet);
18+
if (options.showProgressRateLimit !== undefined) {
19+
setShowProgressRateLimit(options.showProgressRateLimit);
1920
}
20-
statusConfig.showProgress = false;
2121

2222
const workspaceRoot = options.cwd;
2323
const snapshotDir = options.snapshotDir;
@@ -49,7 +49,7 @@ export function main(): void {
4949
const outputFile = path.join(projectRoot, options.output);
5050
const output = fs.openSync(outputFile, 'w');
5151

52-
console.log('Indexing Dir:', projectRoot, ' // version:', projectVersion);
52+
sendStatus(`Indexing ${projectRoot} with version ${projectVersion}`);
5353

5454
try {
5555
let indexer = new Indexer({
@@ -66,14 +66,19 @@ export function main(): void {
6666

6767
indexer.index();
6868
} catch (e) {
69-
console.warn('Experienced Fatal Error While Indexing: Please create an issue:', e);
69+
console.warn(
70+
'\n\nExperienced Fatal Error While Indexing:\nPlease create an issue at github.com/sourcegraph/scip-python:',
71+
e
72+
);
7073
exit(1);
7174
}
7275

7376
fs.close(output);
7477

7578
if (snapshotDir) {
76-
const scipIndex = scip.Index.deserializeBinary(fs.readFileSync(options.output));
79+
sendStatus(`Writing snapshot from index: ${outputFile}`);
80+
81+
const scipIndex = scip.Index.deserializeBinary(fs.readFileSync(outputFile));
7782
for (const doc of scipIndex.documents) {
7883
if (doc.relative_path.startsWith('..')) {
7984
console.log('Skipping Doc:', doc.relative_path);
@@ -90,7 +95,10 @@ export function main(): void {
9095
}
9196
},
9297
(snapshotRoot, options) => {
93-
statusConfig.showProgress = false;
98+
setQuiet(options.quiet);
99+
if (options.showProgressRateLimit !== undefined) {
100+
setShowProgressRateLimit(options.showProgressRateLimit);
101+
}
94102

95103
console.log('... Snapshotting ... ');
96104
const projectName = options.projectName;

‎packages/pyright-scip/src/status.ts

Lines changed: 96 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,103 @@
1-
import ora, { Ora } from 'ora';
1+
const statusConfig = { quiet: true };
22

3-
export const statusConfig = {
4-
showProgress: true,
5-
};
3+
class Logger {
4+
private lastProgressMsg: Date;
65

7-
export function withStatus<T>(msg: string, f: (s: Ora) => T): T {
8-
const spinner = ora({
9-
text: msg,
10-
spinner: 'arc',
11-
interval: 100,
12-
});
6+
/// Minimum number of seconds between progress messages in the output.
7+
public showProgressRateLimit: number;
138

14-
if (statusConfig.showProgress) {
15-
spinner.start();
9+
public depth: number;
10+
11+
constructor(showProgressRateLimit = 1) {
12+
this.depth = 0;
13+
this.showProgressRateLimit = showProgressRateLimit;
14+
this.lastProgressMsg = new Date();
15+
}
16+
17+
private timestamp(): string {
18+
const now = new Date();
19+
return `(${now.toLocaleTimeString('default', {
20+
hour: '2-digit',
21+
minute: '2-digit',
22+
second: '2-digit',
23+
hour12: false,
24+
})}) `;
25+
}
26+
27+
section(msg: string): void {
28+
this.write_message(this.depth, msg);
29+
30+
this.depth += 1;
31+
this.lastProgressMsg = new Date();
32+
}
33+
34+
message(msg: string): void {
35+
this.write_message(this.depth + 1, msg);
36+
}
37+
38+
write_message(depth: number, msg: string): void {
39+
process.stdout.write(this.timestamp());
40+
if (depth > 0) {
41+
process.stdout.write(' '.repeat(this.depth));
42+
}
43+
console.log(msg);
44+
}
45+
46+
complete(): void {
47+
this.depth -= 1;
48+
}
49+
50+
progress(msg: string): void {
51+
const now = new Date();
52+
if (
53+
this.showProgressRateLimit == 0 ||
54+
!this.lastProgressMsg ||
55+
(now.getTime() - this.lastProgressMsg.getTime()) / 1000 > this.showProgressRateLimit
56+
) {
57+
this.lastProgressMsg = now;
58+
this.write_message(this.depth + 1, '- ' + msg);
59+
}
1660
}
61+
}
62+
63+
const logger = new Logger();
64+
65+
export interface StatusUpdater {
66+
/// Send a message at a certain interval (when enabled).
67+
/// Use this to send messages that could be sent many times (which will prevent
68+
/// messages being spammed in the logs)
69+
progress: (msg: any) => void;
70+
71+
/// Messasge will always send a message (when enabled).
72+
/// Do not use within tight loops that could generate thousands of messages,
73+
/// instead use StatusUpdater.progress
74+
message: (msg: any) => void;
75+
}
76+
77+
export function setQuiet(quiet: boolean): void {
78+
statusConfig.quiet = quiet;
79+
}
1780

18-
let v = f(spinner);
81+
export function setShowProgressRateLimit(ratelimit: number): void {
82+
logger.showProgressRateLimit = ratelimit;
83+
}
84+
85+
export function withStatus<T>(msg: string, f: (progress: StatusUpdater) => T): T {
86+
const progress = statusConfig.quiet
87+
? { section: () => {}, complete: () => {}, progress: (_: string) => {}, message: (_: string) => {} }
88+
: logger;
89+
90+
progress.section(msg);
91+
const val = f(progress);
92+
progress.complete();
93+
94+
return val;
95+
}
96+
97+
export function sendStatus(msg: string) {
98+
if (statusConfig.quiet) {
99+
return;
100+
}
19101

20-
spinner.succeed();
21-
return v;
102+
logger.write_message(0, msg);
22103
}

0 commit comments

Comments
 (0)
0