diff --git a/.github/workflows/scip-snapshot.yml b/.github/workflows/scip-snapshot.yml index aeab70e6d..7bc4e1f26 100644 --- a/.github/workflows/scip-snapshot.yml +++ b/.github/workflows/scip-snapshot.yml @@ -1,8 +1,5 @@ name: scip-snapshots -env: - NODE_VERSION: '16.7.0' - on: push: branches: @@ -16,15 +13,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - name: Install asdf. + uses: asdf-vm/actions/setup@v2.1.0 + - name: Cache asdf and asdf-managed tools. + uses: actions/cache@v3.3.1 + id: asdf-cache with: - node-version: ${{ env.NODE_VERSION }} - registry-url: 'https://registry.npmjs.org' - + path: ${{ env.ASDF_DIR }} + key: asdf-${{ runner.os}}-${{ hashFiles('**/.tool-versions') }} + - name: Install asdf tools (if not cached). + if: steps.asdf-cache.outputs.cache-hit != 'true' + uses: asdf-vm/actions/install@v2.1.0 - name: Get npm cache directory id: npm-cache - run: | - echo "::set-output name=dir::$(npm config get cache)" + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT - uses: actions/cache@v2 with: path: ${{ steps.npm-cache.outputs.dir }} @@ -33,9 +35,6 @@ jobs: ${{ runner.os }}-node- - run: npm install - - run: cd ./packages/pyright-scip/ && npm install && npm run build - - run: python --version - - run: cd ./packages/pyright-scip/ && npm run check-snapshots diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..cdb9f849c --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +python 3.10.11 +nodejs 18.16.0 diff --git a/README.md b/README.md index c8265158d..e802a95ec 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,87 @@ $ npm install -g @sourcegraph/scip-python scip-python requires Node v16 or newer. See the [Dockerfile](https://github.com/sourcegraph/scip-python/blob/scip/Dockerfile.autoindex) for an exact SHA that is tested. +scip-python uses `pip` to attempt to determine the versions and names of the packages available in your environment. If you do not use pip to install the packages, you can instead use the `--environment` flag to supply a list of packages to use as the environment. This will skip any calls out to pip to determine the state of your env. See [Environment](##-environment) for more information. + + ## Usage ``` $ npm install @sourcegraph/scip-python $ # NOTE: make sure to activate your virtual environment before running -$ scip-python index . --project-name $MY_PROJECT +$ scip-python index . --project-name=$MY_PROJECT $ # Make sure to point towards the sourcegraph instance you're interested in uploading to. $ # more information at https://github.com/sourcegraph/src-cli $ src code-intel upload ``` +### target-only + +To run scip-python over only a particular directory, you can use the `--target-only` flag. Example: + +``` +$ scip-python index . --project-name=$MY_PROJECT --target-only=src/subdir +``` + +### project-namespace + +Additionally, if your project is loaded with some prefix, you can use the `--project-namespace` to put a namespace before all the generated symbols for this project. + +``` +$ scip-python index . --project-name=$MY_PROJECT --project-namespace=implicit.namespace +``` + +Now all symbols will have `implicit.namespace` prepended to their symbol, so that you can use it for cross repository navigation, even if the directory structure in your current project does not explicitly show `implicit/namespace/myproject/__init__.py`. + +## Environment + +The environment file format is a JSON list of `PythonPackage`s. The `PythonPackage` has the following form: + +```json +{ + "name": "PyYAML", + "version": "6.0", + "files": [ + "PyYAML-6.0.dist-info/INSTALLER", + ... + "yaml/__init__.py", + "yaml/composer.py", + "yaml/tokens.py", + ... + ] +}, +``` + +Where: +- `name`: + - The name of the package. Often times this is the same as the module, but is not always the case. + - For example, `PyYAML` is the name of the package, but the module is `yaml` (i.e. `import yaml`). +- `version`: + - The vesion of the package. This is used to generate stable references to external packages. +- `files`: + - A list of all the files that are a member of this package. + - Some packages declare multiple modules, so these should all be included. + +The environment file should be a list of these packages: + +```json +[ + { "name": "PyYAML", "version": "6.0", "files": [...] }, + { "name": "pytorch", "version": "3.0", "files": [..] }, + ... +] +``` + +To use the environment file, you should call scip-python like so: + +``` +$ scip-python index --project-name=$MY_PROJECT --environment=path/to/env.json +``` + +If you're just using pip, this should not be required. We should calculate this from the pip environment. If you experience any bugs, please report them. The goal is that we support standard pip installation without additional configuration. If there is other python tooling that can generate this information, you can file an issue and we'll see if we can support it as well. + ## Sourcegraph Example Configuration Using the usage example above may be quite simple to add a CI pipeline (perhaps using the `sourcegraph/scip-python:autoindex`) image diff --git a/packages/pyright-scip/CHANGELOG.md b/packages/pyright-scip/CHANGELOG.md new file mode 100644 index 000000000..c52bf569f --- /dev/null +++ b/packages/pyright-scip/CHANGELOG.md @@ -0,0 +1,8 @@ +# Release v0.4 + +- remove: `--include` and `--exclude`. Instead use `pyproject.toml` and pyright configuration. +- add: `--target-only` to only emit and parse information related to some subdirectory of your project. Should still be run from root of project. +- add: `--project-namespace` to prefix any definitions in your current project. This can be useful when your package gets installed in some non-standard way and there doesn't have the appropriate prefix that other python packages would import from. +- Now respects pyright config by default (and discovers applicable pyright configuration). +- Updated pyright internal library +- Attempt to capture possible failures in pyright library so some indexing can still be completed. diff --git a/packages/pyright-scip/package-lock.json b/packages/pyright-scip/package-lock.json index d493ae617..c85c45142 100644 --- a/packages/pyright-scip/package-lock.json +++ b/packages/pyright-scip/package-lock.json @@ -9,6 +9,7 @@ "version": "0.3.3", "license": "MIT", "dependencies": { + "@iarna/toml": "2.2.5", "commander": "^9.2.0", "diff": "^5.0.0", "glob": "^7.2.0", @@ -775,6 +776,11 @@ "dev": true, "peer": true }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -10339,6 +10345,11 @@ "dev": true, "peer": true }, + "@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", diff --git a/packages/pyright-scip/package.json b/packages/pyright-scip/package.json index 43a6fd6ad..3c56221a6 100644 --- a/packages/pyright-scip/package.json +++ b/packages/pyright-scip/package.json @@ -8,7 +8,7 @@ "clean": "shx rm -rf ./dist ./out README.md LICENSE.txt", "prepack": "npm run clean && shx cp ../../README.md . && shx cp ../../LICENSE.txt . && npm run build", "check-snapshots": "npm run update-snapshots -- --check", - "update-snapshots": "node ./index.js snapshot-dir snapshots --environment snapshots/testEnv.json --no-progress-bar", + "update-snapshots": "node ./index.js snapshot-dir snapshots --environment snapshots/testEnv.json --quiet", "test": "jest --forceExit --detectOpenHandles", "webpack": "webpack --mode development --progress", "watch": "webpack --mode development --progress --watch" @@ -42,6 +42,7 @@ "webpack-cli": "^4.9.1" }, "dependencies": { + "@iarna/toml": "2.2.5", "commander": "^9.2.0", "diff": "^5.0.0", "glob": "^7.2.0", diff --git a/packages/pyright-scip/snapshots/output/aliased_import/actual.py b/packages/pyright-scip/snapshots/output/aliased_import/actual.py index 8e1121700..a6253ac5f 100644 --- a/packages/pyright-scip/snapshots/output/aliased_import/actual.py +++ b/packages/pyright-scip/snapshots/output/aliased_import/actual.py @@ -11,7 +11,7 @@ # > ``` print(A.SOME_CONSTANT) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/builtin_imports/builtin_imports.py b/packages/pyright-scip/snapshots/output/builtin_imports/builtin_imports.py index 7b939e11d..e14550560 100644 --- a/packages/pyright-scip/snapshots/output/builtin_imports/builtin_imports.py +++ b/packages/pyright-scip/snapshots/output/builtin_imports/builtin_imports.py @@ -38,7 +38,7 @@ # > ``` print(re, Callable, Optional) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/class_nohint/class_nohint.py b/packages/pyright-scip/snapshots/output/class_nohint/class_nohint.py index 545b17d76..7cec6c45e 100644 --- a/packages/pyright-scip/snapshots/output/class_nohint/class_nohint.py +++ b/packages/pyright-scip/snapshots/output/class_nohint/class_nohint.py @@ -53,7 +53,7 @@ def something(self): # > ``` # ^^^^ definition snapshot-util 0.1 class_nohint/Example#something().(self) print(self.x) -# ^^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +# ^^^^^ reference python-stdlib 3.11 builtins/print(). # external documentation ```python # > (function) def print( # > *values: object, @@ -66,7 +66,7 @@ def something(self): # ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) # ^ reference snapshot-util 0.1 class_nohint/Example#x. print(self.y) -# ^^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +# ^^^^^ reference python-stdlib 3.11 builtins/print(). # ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) # ^ reference snapshot-util 0.1 class_nohint/Example#y. diff --git a/packages/pyright-scip/snapshots/output/comprehensions/comp.py b/packages/pyright-scip/snapshots/output/comprehensions/comp.py index 8c696fde0..ab89f8f67 100644 --- a/packages/pyright-scip/snapshots/output/comprehensions/comp.py +++ b/packages/pyright-scip/snapshots/output/comprehensions/comp.py @@ -75,7 +75,7 @@ def something(x): # ^^^ definition snapshot-util 0.1 comp/var. # ^^^^ reference snapshot-util 0.1 comp/asdf. print(var) -# ^^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +# ^^^^^ reference python-stdlib 3.11 builtins/print(). # external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/dunder_vars/__main__.py b/packages/pyright-scip/snapshots/output/dunder_vars/__main__.py index ee44e14a6..ffcdc2677 100644 --- a/packages/pyright-scip/snapshots/output/dunder_vars/__main__.py +++ b/packages/pyright-scip/snapshots/output/dunder_vars/__main__.py @@ -8,7 +8,7 @@ # > __name__: str # > ``` print("main") -# ^^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +# ^^^^^ reference python-stdlib 3.11 builtins/print(). # external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/f_string/fstring.py b/packages/pyright-scip/snapshots/output/f_string/fstring.py index c4d954cc5..a40594ee7 100644 --- a/packages/pyright-scip/snapshots/output/f_string/fstring.py +++ b/packages/pyright-scip/snapshots/output/f_string/fstring.py @@ -8,7 +8,7 @@ # > ``` print(f"var: hello {var}") -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/file_from_module_import/abc/file.py b/packages/pyright-scip/snapshots/output/file_from_module_import/abc/file.py index bb8df66dd..4da5f0c4d 100644 --- a/packages/pyright-scip/snapshots/output/file_from_module_import/abc/file.py +++ b/packages/pyright-scip/snapshots/output/file_from_module_import/abc/file.py @@ -6,7 +6,7 @@ # ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: print(nested_file.X) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/file_from_module_import/file_from_module.py b/packages/pyright-scip/snapshots/output/file_from_module_import/file_from_module.py index 9fbeee51a..f9e22416b 100644 --- a/packages/pyright-scip/snapshots/output/file_from_module_import/file_from_module.py +++ b/packages/pyright-scip/snapshots/output/file_from_module_import/file_from_module.py @@ -6,7 +6,7 @@ # ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: print(nested_file.X) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/nested_items/src/importer.py b/packages/pyright-scip/snapshots/output/nested_items/src/importer.py index f22d6d5b9..cd90cbb8b 100644 --- a/packages/pyright-scip/snapshots/output/nested_items/src/importer.py +++ b/packages/pyright-scip/snapshots/output/nested_items/src/importer.py @@ -10,7 +10,7 @@ # ^^^^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow# print(SuchNestedMuchWow().class_item) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, @@ -23,11 +23,11 @@ # ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow# # ^^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow#class_item. print(AnotherNestedMuchWow().other_item) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). # ^^^^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow# # ^^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. print(InitClass().init_item) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). # ^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar`/InitClass# # ^^^^^^^^^ reference snapshot-util 0.1 `src.foo.bar`/InitClass#init_item. diff --git a/packages/pyright-scip/snapshots/output/nested_items/src/long_importer.py b/packages/pyright-scip/snapshots/output/nested_items/src/long_importer.py index 7a487298e..bc76513e4 100644 --- a/packages/pyright-scip/snapshots/output/nested_items/src/long_importer.py +++ b/packages/pyright-scip/snapshots/output/nested_items/src/long_importer.py @@ -5,7 +5,7 @@ # ^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/__init__: print(foo.bar.baz.mod.SuchNestedMuchWow) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/request_goofiness/goofy.py b/packages/pyright-scip/snapshots/output/request_goofiness/goofy.py index 01c76b5c4..2306e980e 100644 --- a/packages/pyright-scip/snapshots/output/request_goofiness/goofy.py +++ b/packages/pyright-scip/snapshots/output/request_goofiness/goofy.py @@ -5,7 +5,7 @@ # ^^^^^^^^ reference requests 2.0.0 requests/__init__: print(requests.get("https://sourcegraph.com")) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/unique/builtin_import_refs.py b/packages/pyright-scip/snapshots/output/unique/builtin_import_refs.py index 1c7e7282f..39a77a9e2 100644 --- a/packages/pyright-scip/snapshots/output/unique/builtin_import_refs.py +++ b/packages/pyright-scip/snapshots/output/unique/builtin_import_refs.py @@ -32,7 +32,7 @@ # > ``` print(Any) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/unique/property_access.py b/packages/pyright-scip/snapshots/output/unique/property_access.py index 87ac02a54..1db148992 100644 --- a/packages/pyright-scip/snapshots/output/unique/property_access.py +++ b/packages/pyright-scip/snapshots/output/unique/property_access.py @@ -86,7 +86,7 @@ def nested(): # ^ definition local 0 # ^^ reference snapshot-util 0.1 property_access/usage().(xs) print(x.prop_ref) -# ^^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +# ^^^^^ reference python-stdlib 3.11 builtins/print(). # external documentation ```python # > (function) def print( # > *values: object, diff --git a/packages/pyright-scip/snapshots/output/unresolved_import/unresolved.py b/packages/pyright-scip/snapshots/output/unresolved_import/unresolved.py index c47b33fcb..602259f89 100644 --- a/packages/pyright-scip/snapshots/output/unresolved_import/unresolved.py +++ b/packages/pyright-scip/snapshots/output/unresolved_import/unresolved.py @@ -6,7 +6,7 @@ # documentation (module): this_is_not_real [unable to re... print(this_is_not_real.x) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). #external documentation ```python # > (function) def print( # > *values: object, @@ -18,7 +18,7 @@ # > ``` # ^^^^^^^^^^^^^^^^ reference local 0 print(this_is_not_real.x) -#^^^^ reference python-stdlib 3.11 builtins/__init__:print(). +#^^^^ reference python-stdlib 3.11 builtins/print(). # ^^^^^^^^^^^^^^^^ reference local 0 diff --git a/packages/pyright-scip/src/MainCommand.test.ts b/packages/pyright-scip/src/MainCommand.test.ts index 9ae0d5b04..5d43310a1 100644 --- a/packages/pyright-scip/src/MainCommand.test.ts +++ b/packages/pyright-scip/src/MainCommand.test.ts @@ -25,6 +25,8 @@ checkIndexParser([], { }); checkIndexParser(['--cwd', 'qux'], { cwd: 'qux' }); -checkIndexParser(['--no-progress-bar'], { quiet: false }); +checkIndexParser(['--quiet'], { quiet: true }); checkIndexParser(['--show-progress-rate-limit', '120'], { showProgressRateLimit: 120 }); checkIndexParser(['--show-progress-rate-limit', '0.5'], { showProgressRateLimit: 0.5 }); +checkIndexParser(['--target-only', 'foo'], { targetOnly: 'foo' }); +checkIndexParser(['--project-namespace', 'LSP'], { projectNamespace: 'LSP' }); diff --git a/packages/pyright-scip/src/MainCommand.ts b/packages/pyright-scip/src/MainCommand.ts index bba3122eb..97734017f 100644 --- a/packages/pyright-scip/src/MainCommand.ts +++ b/packages/pyright-scip/src/MainCommand.ts @@ -5,13 +5,13 @@ export interface IndexOptions { project: string; projectName: string; projectVersion: string; + projectNamespace?: string; snapshotDir: string; environment?: string; dev: boolean; - include: string; - exclude: string; output: string; cwd: string; + targetOnly?: string; // Progress reporting configuration quiet: boolean; @@ -55,10 +55,11 @@ export function mainCommand( .command('index') .requiredOption('--project-name ', 'the name of the current project, pypi name if applicable') .option('--project-version ', 'the version of the current project, defaults to git revision') + .option('--project-namespace ', 'A prefix to prepend to all module definitions in the current index') .option('--cwd ', 'working directory for executing scip-python', process.cwd()) + .option('--target-only ', 'limit analysis to the following path') .option('--output ', 'path to the output file', DEFAULT_OUTPUT_FILE) .option('--snapshot-dir ', 'the directory to output a snapshot of the SCIP dump') - .option('--no-progress-bar', '(deprecated, use "--quiet")') .option('--quiet', 'run without logging and status information', false) .option( '--show-progress-rate-limit ', @@ -66,8 +67,6 @@ export function mainCommand( parseOptionalNum ) .option('--environment ', 'the environment json file (experimental)') - .option('--include ', 'comma-separated list of patterns to include (experimental)') - .option('--exclude ', 'comma-separated list of patterns to exclude (experimental)') .option('--dev', 'run in developer mode (experimental)', false) .action((parsedOptions) => { indexAction(parsedOptions as IndexOptions); @@ -83,7 +82,6 @@ export function mainCommand( .option('--output ', 'path to the output file', DEFAULT_OUTPUT_FILE) .option('--environment ', 'the environment json file (experimental)') .option('--no-index', 'skip indexing (use existing index.scip)') - .option('--no-progress-bar', '(deprecated, use "--quiet")') .option('--quiet', 'run without logging and status information', false) .option( '--show-progress-rate-limit ', diff --git a/packages/pyright-scip/src/config.ts b/packages/pyright-scip/src/config.ts new file mode 100644 index 000000000..9d21c96c6 --- /dev/null +++ b/packages/pyright-scip/src/config.ts @@ -0,0 +1,445 @@ +import * as TOML from '@iarna/toml'; +import * as JSONC from 'jsonc-parser'; +import { findPythonSearchPaths, getTypeShedFallbackPath } from 'pyright-internal/analyzer/pythonPathUtils'; + +import { CommandLineOptions } from 'pyright-internal/common/commandLineOptions'; +import { ConfigOptions } from 'pyright-internal/common/configOptions'; +import { FullAccessHost } from 'pyright-internal/common/fullAccessHost'; +import { Host } from 'pyright-internal/common/host'; +import { defaultStubsDirectory } from 'pyright-internal/common/pathConsts'; +import { + forEachAncestorDirectory, + combinePaths, + getDirectoryPath, + normalizePath, + isDirectory, + getFileSpec, +} from 'pyright-internal/common/pathUtils'; +import { PyrightFileSystem } from 'pyright-internal/pyrightFileSystem'; +import { ScipConfig } from './lib'; + +const configFileNames = ['scip-pyrightconfig.json', 'pyrightconfig.json']; +const pyprojectTomlName = 'pyproject.toml'; + +export class ScipPyrightConfig { + fs: PyrightFileSystem; + _configFilePath: string | undefined; + _configOptions: ConfigOptions; + + _console: Console = console; + _typeCheckingMode = 'basic'; + + constructor(scipConfig: ScipConfig, fs: PyrightFileSystem) { + this.fs = fs; + + this._configOptions = new ConfigOptions(scipConfig.projectRoot); + this._configOptions.checkOnlyOpenFiles = false; + this._configOptions.indexing = true; + this._configOptions.useLibraryCodeForTypes = true; + } + + getConfigOptions(): ConfigOptions { + const host = new FullAccessHost(this.fs); + + // TODO: This probably should be ScipConfig.workspaceroot or similar? + const options = new CommandLineOptions(process.cwd(), false); + + let config = this._getConfigOptions(host, options); + config.checkOnlyOpenFiles = false; + config.indexing = true; + config.useLibraryCodeForTypes = true; + config.typeshedPath = this._configOptions.typeshedPath || getTypeShedFallbackPath(this.fs); + + return config; + } + + // EVERYTHING BELOW HERE IS COPIED FROM: + // - packages/pyright-internal/src/analyzer/service.ts + // + // These are not exposed and too coupled to the analysis service, + // it doesn't make sense to try and connect them at this time. + private _getConfigOptions(host: Host, commandLineOptions: CommandLineOptions): ConfigOptions { + let projectRoot = commandLineOptions.executionRoot; + let configFilePath: string | undefined; + let pyprojectFilePath: string | undefined; + + if (commandLineOptions.configFilePath) { + // If the config file path was specified, determine whether it's + // a directory (in which case the default config file name is assumed) + // or a file. + configFilePath = combinePaths( + commandLineOptions.executionRoot, + normalizePath(commandLineOptions.configFilePath) + ); + if (!this.fs.existsSync(configFilePath)) { + this._console.info(`Configuration file not found at ${configFilePath}.`); + configFilePath = commandLineOptions.executionRoot; + } else { + if (configFilePath.toLowerCase().endsWith('.json')) { + projectRoot = getDirectoryPath(configFilePath); + } else { + projectRoot = configFilePath; + configFilePath = this._findConfigFile(configFilePath); + if (!configFilePath) { + this._console.info(`Configuration file not found at ${projectRoot}.`); + } + } + } + } else if (projectRoot) { + // In a project-based IDE like VS Code, we should assume that the + // project root directory contains the config file. + configFilePath = this._findConfigFile(projectRoot); + + // If pyright is being executed from the command line, the working + // directory may be deep within a project, and we need to walk up the + // directory hierarchy to find the project root. + if (!configFilePath && !commandLineOptions.fromVsCodeExtension) { + configFilePath = this._findConfigFileHereOrUp(projectRoot); + } + + if (configFilePath) { + projectRoot = getDirectoryPath(configFilePath); + } else { + this._console.log(`No configuration file found.`); + configFilePath = undefined; + } + } + + if (!configFilePath) { + // See if we can find a pyproject.toml file in this directory. + pyprojectFilePath = this._findPyprojectTomlFile(projectRoot); + + if (!pyprojectFilePath && !commandLineOptions.fromVsCodeExtension) { + pyprojectFilePath = this._findPyprojectTomlFileHereOrUp(projectRoot); + } + + if (pyprojectFilePath) { + projectRoot = getDirectoryPath(pyprojectFilePath); + this._console.log(`pyproject.toml file found at ${projectRoot}.`); + } else { + this._console.log(`No pyproject.toml file found.`); + } + } + + const configOptions = new ConfigOptions(projectRoot, this._typeCheckingMode); + const defaultExcludes = ['**/node_modules', '**/__pycache__', '**/.*']; + + if (commandLineOptions.pythonPath) { + configOptions.pythonPath = commandLineOptions.pythonPath; + } + + // The pythonPlatform and pythonVersion from the command-line can be overridden + // by the config file, so initialize them upfront. + configOptions.defaultPythonPlatform = commandLineOptions.pythonPlatform; + configOptions.defaultPythonVersion = commandLineOptions.pythonVersion; + configOptions.ensureDefaultExtraPaths( + this.fs, + commandLineOptions.autoSearchPaths || false, + commandLineOptions.extraPaths + ); + + if (commandLineOptions.fileSpecs.length > 0) { + commandLineOptions.fileSpecs.forEach((fileSpec) => { + configOptions.include.push(getFileSpec(this.fs, projectRoot, fileSpec)); + }); + } + + if (commandLineOptions.excludeFileSpecs.length > 0) { + commandLineOptions.excludeFileSpecs.forEach((fileSpec) => { + configOptions.exclude.push(getFileSpec(this.fs, projectRoot, fileSpec)); + }); + } + + if (commandLineOptions.ignoreFileSpecs.length > 0) { + commandLineOptions.ignoreFileSpecs.forEach((fileSpec) => { + configOptions.ignore.push(getFileSpec(this.fs, projectRoot, fileSpec)); + }); + } + + if (!configFilePath && commandLineOptions.executionRoot) { + if (commandLineOptions.fileSpecs.length === 0) { + // If no config file was found and there are no explicit include + // paths specified, assume the caller wants to include all source + // files under the execution root path. + configOptions.include.push(getFileSpec(this.fs, commandLineOptions.executionRoot, '.')); + } + + if (commandLineOptions.excludeFileSpecs.length === 0) { + // Add a few common excludes to avoid long scan times. + defaultExcludes.forEach((exclude) => { + configOptions.exclude.push(getFileSpec(this.fs, commandLineOptions.executionRoot, exclude)); + }); + } + } + + this._configFilePath = configFilePath || pyprojectFilePath; + + // If we found a config file, parse it to compute the effective options. + let configJsonObj: object | undefined; + if (configFilePath) { + this._console.info(`Loading configuration file at ${configFilePath}`); + configJsonObj = this._parseJsonConfigFile(configFilePath); + } else if (pyprojectFilePath) { + this._console.info(`Loading pyproject.toml file at ${pyprojectFilePath}`); + configJsonObj = this._parsePyprojectTomlFile(pyprojectFilePath); + } + + if (configJsonObj) { + configOptions.initializeFromJson( + configJsonObj, + this._typeCheckingMode, + this._console, + this.fs, + host, + commandLineOptions.diagnosticSeverityOverrides, + commandLineOptions.fileSpecs.length > 0 + ); + + const configFileDir = getDirectoryPath(this._configFilePath!); + + // If no include paths were provided, assume that all files within + // the project should be included. + if (configOptions.include.length === 0) { + this._console.info(`No include entries specified; assuming ${configFileDir}`); + configOptions.include.push(getFileSpec(this.fs, configFileDir, '.')); + } + + // If there was no explicit set of excludes, add a few common ones to avoid long scan times. + if (configOptions.exclude.length === 0) { + defaultExcludes.forEach((exclude) => { + this._console.info(`Auto-excluding ${exclude}`); + configOptions.exclude.push(getFileSpec(this.fs, configFileDir, exclude)); + }); + + if (configOptions.autoExcludeVenv === undefined) { + configOptions.autoExcludeVenv = true; + } + } + } else { + configOptions.autoExcludeVenv = true; + configOptions.applyDiagnosticOverrides(commandLineOptions.diagnosticSeverityOverrides); + } + + // Override the analyzeUnannotatedFunctions setting based on the command-line setting. + if (commandLineOptions.analyzeUnannotatedFunctions !== undefined) { + configOptions.diagnosticRuleSet.analyzeUnannotatedFunctions = + commandLineOptions.analyzeUnannotatedFunctions; + } + + const reportDuplicateSetting = (settingName: string, configValue: number | string | boolean) => { + const settingSource = commandLineOptions.fromVsCodeExtension + ? 'the client settings' + : 'a command-line option'; + this._console.warn( + `The ${settingName} has been specified in both the config file and ` + + `${settingSource}. The value in the config file (${configValue}) ` + + `will take precedence` + ); + }; + + // Apply the command-line options if the corresponding + // item wasn't already set in the config file. Report any + // duplicates. + if (commandLineOptions.venvPath) { + if (!configOptions.venvPath) { + configOptions.venvPath = commandLineOptions.venvPath; + } else { + reportDuplicateSetting('venvPath', configOptions.venvPath); + } + } + + if (commandLineOptions.typeshedPath) { + if (!configOptions.typeshedPath) { + configOptions.typeshedPath = commandLineOptions.typeshedPath; + } else { + reportDuplicateSetting('typeshedPath', configOptions.typeshedPath); + } + } + + configOptions.verboseOutput = commandLineOptions.verboseOutput ?? configOptions.verboseOutput; + configOptions.checkOnlyOpenFiles = !!commandLineOptions.checkOnlyOpenFiles; + configOptions.autoImportCompletions = !!commandLineOptions.autoImportCompletions; + configOptions.indexing = !!commandLineOptions.indexing; + configOptions.taskListTokens = commandLineOptions.taskListTokens; + configOptions.logTypeEvaluationTime = !!commandLineOptions.logTypeEvaluationTime; + configOptions.typeEvaluationTimeThreshold = commandLineOptions.typeEvaluationTimeThreshold; + + // If useLibraryCodeForTypes was not specified in the config, allow the settings + // or command line to override it. + if (configOptions.useLibraryCodeForTypes === undefined) { + configOptions.useLibraryCodeForTypes = !!commandLineOptions.useLibraryCodeForTypes; + } else if (commandLineOptions.useLibraryCodeForTypes !== undefined) { + reportDuplicateSetting('useLibraryCodeForTypes', configOptions.useLibraryCodeForTypes); + } + + if (commandLineOptions.stubPath) { + if (!configOptions.stubPath) { + configOptions.stubPath = commandLineOptions.stubPath; + } else { + reportDuplicateSetting('stubPath', configOptions.stubPath); + } + } + + if (configOptions.stubPath) { + // If there was a stub path specified, validate it. + if (!this.fs.existsSync(configOptions.stubPath) || !isDirectory(this.fs, configOptions.stubPath)) { + this._console.warn(`stubPath ${configOptions.stubPath} is not a valid directory.`); + } + } else { + // If no stub path was specified, use a default path. + configOptions.stubPath = normalizePath(combinePaths(configOptions.projectRoot, defaultStubsDirectory)); + } + + // Do some sanity checks on the specified settings and report missing + // or inconsistent information. + if (configOptions.venvPath) { + if (!this.fs.existsSync(configOptions.venvPath) || !isDirectory(this.fs, configOptions.venvPath)) { + this._console.error(`venvPath ${configOptions.venvPath} is not a valid directory.`); + } + + // venvPath without venv means it won't do anything while resolveImport. + // so first, try to set venv from existing configOption if it is null. if both are null, + // then, resolveImport won't consider venv + configOptions.venv = configOptions.venv ?? this._configOptions.venv; + if (configOptions.venv) { + const fullVenvPath = combinePaths(configOptions.venvPath, configOptions.venv); + + if (!this.fs.existsSync(fullVenvPath) || !isDirectory(this.fs, fullVenvPath)) { + this._console.error( + `venv ${configOptions.venv} subdirectory not found in venv path ${configOptions.venvPath}.` + ); + } else { + const importFailureInfo: string[] = []; + if (findPythonSearchPaths(this.fs, configOptions, host, importFailureInfo) === undefined) { + this._console.error( + `site-packages directory cannot be located for venvPath ` + + `${configOptions.venvPath} and venv ${configOptions.venv}.` + ); + + if (configOptions.verboseOutput) { + importFailureInfo.forEach((diag) => { + this._console.error(` ${diag}`); + }); + } + } + } + } + } + + // Is there a reference to a venv? If so, there needs to be a valid venvPath. + if (configOptions.venv) { + if (!configOptions.venvPath) { + this._console.warn(`venvPath not specified, so venv settings will be ignored.`); + } + } + + if (configOptions.typeshedPath) { + if (!this.fs.existsSync(configOptions.typeshedPath) || !isDirectory(this.fs, configOptions.typeshedPath)) { + this._console.error(`typeshedPath ${configOptions.typeshedPath} is not a valid directory.`); + } + } + + return configOptions; + } + + private _findConfigFile(searchPath: string): string | undefined { + for (const name of configFileNames) { + const fileName = combinePaths(searchPath, name); + if (this.fs.existsSync(fileName)) { + return fileName; + } + } + return undefined; + } + + private _findConfigFileHereOrUp(searchPath: string): string | undefined { + return forEachAncestorDirectory(searchPath, (ancestor) => this._findConfigFile(ancestor)); + } + + private _findPyprojectTomlFile(searchPath: string) { + const fileName = combinePaths(searchPath, pyprojectTomlName); + if (this.fs.existsSync(fileName)) { + return fileName; + } + return undefined; + } + + private _findPyprojectTomlFileHereOrUp(searchPath: string): string | undefined { + return forEachAncestorDirectory(searchPath, (ancestor) => this._findPyprojectTomlFile(ancestor)); + } + + private _parseJsonConfigFile(configPath: string): object | undefined { + return this._attemptParseFile(configPath, (fileContents) => { + const errors: JSONC.ParseError[] = []; + const result = JSONC.parse(fileContents, errors, { allowTrailingComma: true }); + if (errors.length > 0) { + throw new Error('Errors parsing JSON file'); + } + + return result; + }); + } + + private _attemptParseFile( + filePath: string, + parseCallback: (contents: string, attempt: number) => object | undefined + ): object | undefined { + let fileContents = ''; + let parseAttemptCount = 0; + + while (true) { + // Attempt to read the file contents. + try { + fileContents = this.fs.readFileSync(filePath, 'utf8'); + } catch { + this._console.error(`Config file "${filePath}" could not be read.`); + return undefined; + } + + // Attempt to parse the file. + let parseFailed = false; + try { + return parseCallback(fileContents, parseAttemptCount + 1); + } catch (e: any) { + parseFailed = true; + } + + if (!parseFailed) { + break; + } + + // If we attempt to read the file immediately after it was saved, it + // may have been partially written when we read it, resulting in parse + // errors. We'll give it a little more time and try again. + if (parseAttemptCount++ >= 5) { + this._console.error(`Config file "${filePath}" could not be parsed. Verify that format is correct.`); + return undefined; + } + } + + return undefined; + } + + private _parsePyprojectTomlFile(pyprojectPath: string): object | undefined { + return this._attemptParseFile(pyprojectPath, (fileContents, attemptCount) => { + try { + // First, try and load tool.scip section + const configObj = TOML.parse(fileContents); + if (configObj && configObj.tool && (configObj.tool as TOML.JsonMap).scip) { + return (configObj.tool as TOML.JsonMap).scip as object; + } + + // Fall back to tool.pyright section + if (configObj && configObj.tool && (configObj.tool as TOML.JsonMap).pyright) { + return (configObj.tool as TOML.JsonMap).pyright as object; + } + } catch (e: any) { + this._console.error(`Pyproject file parse attempt ${attemptCount} error: ${JSON.stringify(e)}`); + throw e; + } + + this._console.error(`Pyproject file "${pyprojectPath}" is missing "[tool.pyright]" section.`); + return undefined; + }); + } +} diff --git a/packages/pyright-scip/src/indexer.ts b/packages/pyright-scip/src/indexer.ts index e6c664c5e..5266d293a 100644 --- a/packages/pyright-scip/src/indexer.ts +++ b/packages/pyright-scip/src/indexer.ts @@ -5,21 +5,20 @@ import { Program } from 'pyright-internal/analyzer/program'; import { ImportResolver } from 'pyright-internal/analyzer/importResolver'; import { createFromRealFileSystem } from 'pyright-internal/common/realFileSystem'; import { ConfigOptions } from 'pyright-internal/common/configOptions'; -import { IndexResults } from 'pyright-internal/languageService/documentSymbolProvider'; import { TreeVisitor } from './treeVisitor'; import { FullAccessHost } from 'pyright-internal/common/fullAccessHost'; import * as url from 'url'; import { ScipConfig } from './lib'; import { SourceFile } from 'pyright-internal/analyzer/sourceFile'; import { Counter } from './lsif-typescript/Counter'; -import { getTypeShedFallbackPath } from 'pyright-internal/analyzer/pythonPathUtils'; import { PyrightFileSystem } from 'pyright-internal/pyrightFileSystem'; import getEnvironment from './virtualenv/environment'; import { version } from 'package.json'; -import { getFileSpec } from 'pyright-internal/common/pathUtils'; import { FileMatcher } from './FileMatcher'; import { sendStatus, StatusUpdater, withStatus } from './status'; import { scip } from './scip'; +import { ScipPyrightConfig } from './config'; +import { setProjectNamespace } from './symbols'; export class Indexer { program: Program; @@ -39,36 +38,37 @@ export class Indexer { // have the same methods of configuring, you might just want to change the include/exclude) // // private _getConfigOptions(host: Host, commandLineOptions: CommandLineOptions): ConfigOptions { - this.pyrightConfig = new ConfigOptions(scipConfig.projectRoot); - this.pyrightConfig.checkOnlyOpenFiles = false; - this.pyrightConfig.indexing = true; - this.pyrightConfig.useLibraryCodeForTypes = true; - - const fs = new PyrightFileSystem(createFromRealFileSystem()); - this.pyrightConfig.typeshedPath = getTypeShedFallbackPath(fs); - - if (this.scipConfig.include) { - this.pyrightConfig.include = this.scipConfig.include - .split(',') - .map((pathspec) => getFileSpec(fs, process.cwd(), pathspec)); - } else { - this.pyrightConfig.include = [getFileSpec(fs, process.cwd(), '.')]; - } + let fs = new PyrightFileSystem(createFromRealFileSystem()); - if (this.scipConfig.exclude) { - this.pyrightConfig.exclude = this.scipConfig.exclude - .split(',') - .map((pathspec) => getFileSpec(fs, process.cwd(), pathspec)); - } + let config = new ScipPyrightConfig(scipConfig, fs); + this.pyrightConfig = config.getConfigOptions(); const matcher = new FileMatcher(this.pyrightConfig, fs); + this.projectFiles = new Set(matcher.matchFiles(this.pyrightConfig.include, this.pyrightConfig.exclude)); + if (scipConfig.targetOnly) { + scipConfig.targetOnly = path.resolve(scipConfig.targetOnly); + const targetFiles: Set = new Set(); + for (const file of this.projectFiles) { + if (file.startsWith(scipConfig.targetOnly)) { + targetFiles.add(file); + } + } + + this.projectFiles = targetFiles; + } + + console.log('Total Project Files', this.projectFiles.size); const host = new FullAccessHost(fs); this.importResolver = new ImportResolver(fs, this.pyrightConfig, host); this.program = new Program(this.importResolver, this.pyrightConfig); this.program.setTrackedFiles([...this.projectFiles]); + + if (scipConfig.projectNamespace) { + setProjectNamespace(scipConfig.projectName, this.scipConfig.projectNamespace!); + } } public index(): void { @@ -77,8 +77,26 @@ export class Indexer { onCancellationRequested: Event.None, }; + let failedAnalysis = 0; + let safe_analyze = () => { + try { + return this.program.analyze({ openFilesTimeInMs: 10000, noOpenFilesTimeInMs: 10000 }); + } catch (e) { + // Allow 100 failed attempts before we give up analysis. + // This shouldn't happen often because it means there's a bug in pyright that + // completely stops execution. You'll at least get some output even if it is failing. + sendStatus(` Analysis partially failed with (${failedAnalysis}/100): ${e}`); + if (failedAnalysis++ < 100) { + return true; + } else { + sendStatus(` Cancelling analysis, but continuing to write index. Please file an issue`); + return false; + } + } + }; + const analyzer_fn = (progress: StatusUpdater) => { - while (this.program.analyze({ openFilesTimeInMs: 10000, noOpenFilesTimeInMs: 10000 })) { + while (safe_analyze()) { const filesCompleted = this.program.getFileCount() - this.program.getFilesToAnalyzeCount(); const filesTotal = this.program.getFileCount(); progress.message(`${filesCompleted} / ${filesTotal}`); @@ -97,7 +115,7 @@ export class Indexer { this.scipConfig.writeIndex( new scip.Index({ metadata: new scip.Metadata({ - project_root: url.pathToFileURL(this.scipConfig.workspaceRoot).toString(), + project_root: url.pathToFileURL(this.getProjectRoot()).toString(), text_document_encoding: scip.TextEncoding.UTF8, tool_info: new scip.ToolInfo({ name: 'scip-python', @@ -148,7 +166,7 @@ export class Indexer { const filepath = sourceFile.getFilePath(); let doc = new scip.Document({ - relative_path: path.relative(this.scipConfig.workspaceRoot, filepath), + relative_path: path.relative(this.getProjectRoot(), filepath), }); const parseResults = sourceFile.getParseResults(); @@ -196,4 +214,12 @@ export class Indexer { sendStatus(`Sucessfully wrote SCIP index to ${this.scipConfig.output}`); } + + private getProjectRoot(): string { + if (this.scipConfig.targetOnly && this.scipConfig.targetOnly !== '') { + return this.scipConfig.targetOnly; + } else { + return this.scipConfig.workspaceRoot; + } + } } diff --git a/packages/pyright-scip/src/lib.ts b/packages/pyright-scip/src/lib.ts index 5a1f7c70b..36f0c5814 100644 --- a/packages/pyright-scip/src/lib.ts +++ b/packages/pyright-scip/src/lib.ts @@ -10,7 +10,7 @@ import { IndexOptions } from './MainCommand'; export interface ScipConfig extends IndexOptions { /** - * The directory where to generate the dump.lsif-typed file. + * The directory where to generate the index.scip file. * * All `Document.relative_path` fields will be relative paths to this directory. */ diff --git a/packages/pyright-scip/src/symbols.ts b/packages/pyright-scip/src/symbols.ts index b3a827cbb..a3b3d5b15 100644 --- a/packages/pyright-scip/src/symbols.ts +++ b/packages/pyright-scip/src/symbols.ts @@ -1,34 +1,79 @@ import { ParseNode } from 'pyright-internal/parser/parseNodes'; import { Counter } from './lsif-typescript/Counter'; -import { metaDescriptor, packageDescriptor, typeDescriptor } from './lsif-typescript/Descriptor'; +import { + metaDescriptor, + methodDescriptor, + packageDescriptor, + parameterDescriptor, + termDescriptor, + typeDescriptor, +} from './lsif-typescript/Descriptor'; import { ScipSymbol } from './ScipSymbol'; import PythonPackage from './virtualenv/PythonPackage'; -export function pythonModule(pythonPackage: PythonPackage, moduleName: string): ScipSymbol { - return ScipSymbol.global( - ScipSymbol.global(ScipSymbol.package(pythonPackage.name, pythonPackage.version), packageDescriptor(moduleName)), - metaDescriptor('__init__') - ); +let namespaces: Map = new Map(); + +export function setProjectNamespace(packageName: string, projectName: string): void { + namespaces.set(packageName, projectName); } export function makePackage(pythonPackage: PythonPackage): ScipSymbol { return ScipSymbol.package(pythonPackage.name, pythonPackage.version); } -export function makeModule(moduleName: string, pythonPackage: PythonPackage): ScipSymbol { +export function makeModule(pythonPackage: PythonPackage, moduleName: string): ScipSymbol { + let ns = namespaces.get(pythonPackage.name); + if (ns) { + moduleName = ns + '.' + moduleName; + } + + return ScipSymbol.global(makePackage(pythonPackage), packageDescriptor(moduleName)); +} + +export function makeModuleInit(pythonPackage: PythonPackage, moduleName: string): ScipSymbol { + let ns = namespaces.get(pythonPackage.name); + if (ns) { + moduleName = ns + '.' + moduleName; + } + return ScipSymbol.global( ScipSymbol.global(makePackage(pythonPackage), packageDescriptor(moduleName)), metaDescriptor('__init__') ); } -export function makeClass(pythonPackage: PythonPackage, moduleName: string, name: string) { +export function makeClass(pythonPackage: PythonPackage, moduleName: string, name: string): ScipSymbol { + let ns = namespaces.get(pythonPackage.name); + if (ns) { + moduleName = ns + '.' + moduleName; + } + return ScipSymbol.global( ScipSymbol.global(makePackage(pythonPackage), packageDescriptor(moduleName)), typeDescriptor(name) ); } +export function makeParameter(sym: ScipSymbol, name: string): ScipSymbol { + return ScipSymbol.global(sym, parameterDescriptor(name)); +} + +export function makeMeta(sym: ScipSymbol, meta: string): ScipSymbol { + return ScipSymbol.global(sym, metaDescriptor(meta)); +} + +export function makeMethod(parent: ScipSymbol, name: string): ScipSymbol { + return ScipSymbol.global(parent, methodDescriptor(name)); +} + +export function makeType(parent: ScipSymbol, name: string): ScipSymbol { + return ScipSymbol.global(parent, typeDescriptor(name)); +} + +export function makeTerm(parent: ScipSymbol, name: string): ScipSymbol { + return ScipSymbol.global(parent, termDescriptor(name)); +} + export function make(_node: ParseNode, counter: Counter): ScipSymbol { return ScipSymbol.local(counter.next()); } diff --git a/packages/pyright-scip/src/treeVisitor.ts b/packages/pyright-scip/src/treeVisitor.ts index 2b61f042b..42fcd79e7 100644 --- a/packages/pyright-scip/src/treeVisitor.ts +++ b/packages/pyright-scip/src/treeVisitor.ts @@ -26,14 +26,6 @@ import { import * as ModifiedTypeUtils from './ModifiedTypeUtils'; import { scip } from './scip'; import * as Symbols from './symbols'; -import { - metaDescriptor, - methodDescriptor, - packageDescriptor, - parameterDescriptor, - termDescriptor, - typeDescriptor, -} from './lsif-typescript/Descriptor'; import { ScipSymbol } from './ScipSymbol'; import { Position } from './lsif-typescript/Position'; import { Range } from './lsif-typescript/Range'; @@ -61,7 +53,6 @@ import { Event } from 'vscode-languageserver'; import { HoverResults } from 'pyright-internal/languageService/hoverProvider'; import { convertDocStringToMarkdown } from 'pyright-internal/analyzer/docStringConversion'; import { assert } from 'pyright-internal/common/debug'; -import { createTracePrinter } from 'pyright-internal/analyzer/tracePrinter'; import { getClassFieldsRecursive } from 'pyright-internal/analyzer/typeUtils'; // Useful functions for later, but haven't gotten far enough yet to use them. @@ -83,8 +74,8 @@ function softAssert(expression: any, message: string, ...exprs: any) { return expression; } +// const _printer = createTracePrinter([process.cwd()]); const _msgs = new Set(); -const _printer = createTracePrinter([process.cwd()]); const _transform = function (exprs: any[]): any[] { const result: any[] = []; for (let i = 0; i < exprs.length; i++) { @@ -193,6 +184,8 @@ export class TreeVisitor extends ParseTreeWalker { log.info('=> Working file:', config.sourceFile.getFilePath(), '<=='); + // TODO: This should happen earlier, a bit weird to throw an error this deep + // in the traversal if (!this.config.scipConfig.projectName) { throw 'Must have project name'; } @@ -235,7 +228,7 @@ export class TreeVisitor extends ParseTreeWalker { const pythonPackage = this.getPackageInfo(node, fileInfo.moduleName); if (pythonPackage) { if (softAssert(pythonPackage === this.projectPackage, 'expected pythonPackage to be this.projectPackage')) { - const symbol = Symbols.makeModule(fileInfo.moduleName, pythonPackage); + const symbol = Symbols.makeModuleInit(pythonPackage, fileInfo.moduleName); this.document.occurrences.push( new scip.Occurrence({ @@ -522,13 +515,13 @@ export class TreeVisitor extends ParseTreeWalker { importInfo.resolvedPaths[0] && path.resolve(importInfo.resolvedPaths[0]).startsWith(this.cwd) ) { - const symbol = Symbols.makeModule(moduleName, this.projectPackage); + const symbol = Symbols.makeModuleInit(this.projectPackage, moduleName); this.pushNewOccurrence(node.module, symbol); } else { const pythonPackage = this.moduleNameNodeToPythonPackage(node.module); if (pythonPackage) { - const symbol = Symbols.makeModule(moduleName, pythonPackage); + const symbol = Symbols.makeModuleInit(pythonPackage, moduleName); this.pushNewOccurrence(node.module, symbol); } else { // For python packages & modules that we cannot resolve, @@ -582,8 +575,6 @@ export class TreeVisitor extends ParseTreeWalker { } private emitDeclaration(node: NameNode, decl: Declaration): boolean { - log.debug(' emitDeclaration:'); - log.debug(' decl:', () => _printer.print(decl)); const parent = node.parent!; if (!decl.node) { @@ -601,7 +592,6 @@ export class TreeVisitor extends ParseTreeWalker { } const isDefinition = decl.node.id === parent.id; - log.debug(' node:', ParseTreeUtils.printParseNodeType(decl.node.nodeType), isDefinition); const builtinType = this.evaluator.getBuiltInType(node, node.value); if (this.isStdlib(decl, builtinType)) { @@ -630,7 +620,7 @@ export class TreeVisitor extends ParseTreeWalker { assert(declNode != node.parent, 'Must not be the definition'); assert(pythonPackage, 'Must have a python package: ' + moduleName); - return Symbols.makeModule(moduleName, pythonPackage); + return Symbols.makeModuleInit(pythonPackage, moduleName); }); // TODO: We could maybe cache this to not always be asking for these names & decls @@ -869,8 +859,6 @@ export class TreeVisitor extends ParseTreeWalker { } override visitName(node: NameNode): boolean { - log.debug('\nVisiting Name:', node.value); - if (!node.parent) { throw `No parent for named node: ${node.token.value}`; } @@ -879,10 +867,8 @@ export class TreeVisitor extends ParseTreeWalker { return true; } - const parent = node.parent; const decls = this.evaluator.getDeclarationsForNameNode(node) || []; - log.debug(' ParentParsenodeType:', ParseTreeUtils.printParseNodeType, parent.nodeType); if (decls.length === 0) { return this.emitNameWithoutDeclaration(node); } @@ -1024,14 +1010,9 @@ export class TreeVisitor extends ParseTreeWalker { return symbol; } case TypeCategory.Module: { - const symbol = ScipSymbol.global( - ScipSymbol.package(builtinType.moduleName, this.stdlibPackage.version), - metaDescriptor('__init__') - ); - + const symbol = Symbols.makeModuleInit(this.stdlibPackage, builtinType.moduleName); this.pushNewOccurrence(node, symbol); - // const pythonPackage = this.getPackageInfo(node, builtinType.moduleName)!; this.emitExternalSymbolInformation(node, symbol, []); return symbol; } @@ -1069,12 +1050,9 @@ export class TreeVisitor extends ParseTreeWalker { case ParseNodeType.Module: { moduleName = getFileInfo(node).moduleName; if (moduleName === 'builtins') { - return Symbols.makeModule('builtins', this.stdlibPackage); + return Symbols.makeModule(this.stdlibPackage, 'builtins'); } else { - return ScipSymbol.global( - ScipSymbol.package(pythonPackage.name, pythonPackage.version), - packageDescriptor(moduleName) - ); + return Symbols.makeModule(pythonPackage, moduleName); } } case ParseNodeType.ModuleName: { @@ -1095,12 +1073,9 @@ export class TreeVisitor extends ParseTreeWalker { pythonPackage = this.stdlibPackage; } - return ScipSymbol.global( - ScipSymbol.global( - ScipSymbol.package(pythonPackage.name, pythonPackage.version), - packageDescriptor(node.nameParts.map((namePart) => namePart.value).join('.')) - ), - metaDescriptor('__init__') + return Symbols.makeModuleInit( + pythonPackage, + node.nameParts.map((namePart) => namePart.value).join('.') ); } case ParseNodeType.MemberAccess: { @@ -1113,27 +1088,18 @@ export class TreeVisitor extends ParseTreeWalker { return ScipSymbol.local(this.counter.next()); } - return ScipSymbol.global(this.getScipSymbol(node.parent!), parameterDescriptor(node.name.value)); + return Symbols.makeParameter(this.getScipSymbol(node.parent!), node.name.value); } case ParseNodeType.Class: { - return ScipSymbol.global( - this.getScipSymbol(node.parent!), - typeDescriptor((node as ClassNode).name.value) - ); + return Symbols.makeType(this.getScipSymbol(node.parent!), (node as ClassNode).name.value); } case ParseNodeType.Function: { let cls = ParseTreeUtils.getEnclosingClass(node, false); if (cls) { - return ScipSymbol.global( - this.getScipSymbol(cls), - methodDescriptor((node as FunctionNode).name!.value) - ); + return Symbols.makeMethod(this.getScipSymbol(cls), (node as FunctionNode).name!.value); } - return ScipSymbol.global( - this.getScipSymbol(node.parent!), - methodDescriptor((node as FunctionNode).name!.value) - ); + return Symbols.makeMethod(this.getScipSymbol(node.parent!), (node as FunctionNode).name!.value); } case ParseNodeType.Suite: { if (node.parent) { @@ -1142,7 +1108,7 @@ export class TreeVisitor extends ParseTreeWalker { // TODO: Not sure what to do about this... // I don't know if we ever need to include this at all. - return ScipSymbol.global(this.getScipSymbol(node.parent!), metaDescriptor('#')); + return Symbols.makeMeta(this.getScipSymbol(node.parent!), '#'); } case ParseNodeType.Name: { const parent = node.parent; @@ -1160,15 +1126,12 @@ export class TreeVisitor extends ParseTreeWalker { return this.getSymbolOnce(node, () => { const pythonPackage = this.getPackageInfo(node, bound.details.moduleName)!; - let symbol = ScipSymbol.global( - ScipSymbol.global( - ScipSymbol.global( - ScipSymbol.package(pythonPackage.name, pythonPackage.version), - packageDescriptor(bound.details.moduleName) - ), - typeDescriptor(bound.details.name) + let symbol = Symbols.makeTerm( + Symbols.makeType( + Symbols.makeModule(pythonPackage, bound.details.moduleName), + bound.details.name ), - termDescriptor(node.value) + node.value ); // TODO: We might not want to do this if it's not the definition? @@ -1199,17 +1162,14 @@ export class TreeVisitor extends ParseTreeWalker { } } - return ScipSymbol.global( - this.getScipSymbol(enclosingSuite || parent), - termDescriptor((node as NameNode).value) - ); + return Symbols.makeTerm(this.getScipSymbol(enclosingSuite || parent), (node as NameNode).value); } case ParseNodeType.TypeAnnotation: { switch (node.valueExpression.nodeType) { case ParseNodeType.Name: { - return ScipSymbol.global( + return Symbols.makeTerm( this.getScipSymbol(ParseTreeUtils.getEnclosingSuite(node) || node.parent!), - termDescriptor(node.valueExpression.value) + node.valueExpression.value ); } default: { @@ -1219,10 +1179,10 @@ export class TreeVisitor extends ParseTreeWalker { } } case ParseNodeType.FunctionAnnotation: { - return ScipSymbol.global( + return Symbols.makeTerm( this.getScipSymbol(node.parent!), // Descriptor.term((node as TypeAnnotationNode).typeAnnotation) - termDescriptor('FuncAnnotation') + 'FuncAnnotation' ); } case ParseNodeType.Decorator: { @@ -1244,7 +1204,7 @@ export class TreeVisitor extends ParseTreeWalker { return this.getScipSymbol(node.parent!); } case ParseNodeType.ImportAs: { - return Symbols.pythonModule(pythonPackage, moduleName); + return Symbols.makeModuleInit(pythonPackage, moduleName); } case ParseNodeType.ImportFrom: { const importPackage = this.moduleNameNodeToPythonPackage(node.module); @@ -1283,9 +1243,9 @@ export class TreeVisitor extends ParseTreeWalker { const pythonPackage = this.moduleNameNodeToPythonPackage(parent.module) || this.projectPackage; - return Symbols.makeModule( - [...parent.module.nameParts, node.name].map((part) => part.value).join('.'), - pythonPackage + return Symbols.makeModuleInit( + pythonPackage, + [...parent.module.nameParts, node.name].map((part) => part.value).join('.') ); } } @@ -1344,6 +1304,9 @@ export class TreeVisitor extends ParseTreeWalker { // Take a `Type` from pyright and turn that into an LSIF symbol. private typeToSymbol(node: NameNode, declNode: ParseNode, typeObj: Type): ScipSymbol { if (Types.isFunction(typeObj)) { + // TODO: Possibly worth checking for parent declarations. + // I'm not sure if that will actually work though for types. + const decl = typeObj.details.declaration; if (!decl) { // throw 'Unhandled missing declaration for type: function'; @@ -1369,18 +1332,12 @@ export class TreeVisitor extends ParseTreeWalker { classType.details.name ); - return ScipSymbol.global(symbol, methodDescriptor(node.value)); + return Symbols.makeMethod(symbol, node.value); } - // return ScipSymbol.global(this.makeScipSymbol( + return ScipSymbol.local(this.counter.next()); } else { - return ScipSymbol.global( - ScipSymbol.global( - ScipSymbol.package(pythonPackage?.name, pythonPackage?.version), - packageDescriptor(typeObj.details.moduleName) - ), - methodDescriptor(node.value) - ); + return Symbols.makeMethod(Symbols.makeModule(pythonPackage, typeObj.details.moduleName), node.value); } } else if (Types.isClass(typeObj)) { const pythonPackage = this.getPackageInfo(node, typeObj.details.moduleName)!; @@ -1393,10 +1350,7 @@ export class TreeVisitor extends ParseTreeWalker { throw 'typevar'; } else if (Types.isModule(typeObj)) { const pythonPackage = this.getPackageInfo(node, typeObj.moduleName)!; - return ScipSymbol.global( - ScipSymbol.package(typeObj.moduleName, pythonPackage.version), - metaDescriptor('__init__') - ); + return Symbols.makeModuleInit(pythonPackage, typeObj.moduleName); } else if (Types.isOverloadedFunction(typeObj)) { if (!typeObj.overloads) { softAssert(false, "Didn't think it would be possible to have overloaded w/ no overloads");