8000 Typecheck typeshed's code with pyright (#9793) · python/typeshed@75f47d0 · GitHub
[go: up one dir, main page]
Skip to content < 8000 link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog.f8fba3bd67fe74f9227b.module.css" />

Commit 75f47d0

Browse files
AvasamAlexWaygoodJelleZijlstra
authored
Typecheck typeshed's code with pyright (#9793)
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com> Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
1 parent b0dc6a3 commit 75f47d0

12 files changed

+125
-32
lines changed

‎.github/workflows/typecheck_typeshed_code.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,31 @@ jobs:
4040
cache-dependency-path: requirements-tests.txt
4141
- run: pip install -r requirements-tests.txt
4242
- run: python ./tests/typecheck_typeshed.py --platform=${{ matrix.platform }}
43+
pyright:
44+
name: Run pyright against the scripts and tests directories
45+
runs-on: ubuntu-latest
46+
strategy:
47+
matrix:
48+
python-platform: ["Linux", "Windows"]
49+
fail-fast: false
50+
steps:
51+
- uses: actions/checkout@v3
52+
- uses: actions/setup-python@v4
53+
with:
54+
python-version: "3.9"
55+
cache: pip
56+
cache-dependency-path: requirements-tests.txt
57+
- run: pip install -r requirements-tests.txt
58+
- name: Get pyright version
59+
uses: SebRollen/toml-action@v1.0.2
60+
id: pyright_version
61+
with:
62+
file: "pyproject.toml"
63+
field: "tool.typeshed.pyright_version"
64+
- name: Run pyright on typeshed
65+
uses: jakebailey/pyright-action@v1
66+
with:
67+
version: ${{ steps.pyright_version.outputs.value }}
68+
python-platform: ${{ matrix.python-platform }}
69+
python-version: "3.9"
70+
project: ./pyrightconfig.scripts_and_tests.json

‎pyrightconfig.scripts_and_tests.json

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
3+
"typeshedPath": ".",
4+
"include": [
5+
"scripts",
6+
"tests",
7+
],
8+
"typeCheckingMode": "strict",
9+
// Runtime libraries used by typeshed are not all py.typed
10+
"useLibraryCodeForTypes": true,
11+
// More of a lint. Unwanted for typeshed's own code.
12+
"reportImplicitStringConcatenation": "none",
13+
// Extra strict settings
14+
"reportMissingModuleSource": "error",
15+
"reportShadowedImports": "error",
16+
"reportCallInDefaultInitializer": "error",
17+
"reportPropertyTypeMismatch": "error",
18+
"reportUninitializedInstanceVariable": "error",
19+
"reportUnnecessaryTypeIgnoreComment": "error" 8000 ,
20+
// Leave "type: ignore" comments to mypy
21+
"enableTypeIgnoreComments": false,
22+
// Too strict
23+
"reportMissingSuperCall": "none",
24+
}

‎requirements-tests.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@ pyyaml==6.0
1515
termcolor>=2
1616
tomli==2.0.1
1717
tomlkit==0.11.6
18-
types-pyyaml
18+
types-pyyaml>=6.0.12.7
1919
types-setuptools
2020
typing-extensions

‎scripts/runtests.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import re
88
import subprocess
99
import sys
10+
from collections.abc import Iterable
1011
from pathlib import Path
11-
from typing import Iterable
1212

1313
try:
1414
from termcolor import colored
@@ -176,7 +176,7 @@ def main() -> None:
176176
print("stubtest:", _SKIPPED)
177177
else:
178178
print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
179-
if pytype_result is None:
179+
if not pytype_result:
180180
print("pytype:", _SKIPPED)
181181
else:
182182
print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)

‎scripts/stubsabot.py

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ async def get_github_repo_info(session: aiohttp.ClientSession, pypi_info: PypiIn
241241
github_tags_info_url = f"https://api.github.com/repos/{url_path}/tags"
242242
async with session.get(github_tags_info_url, headers=get_github_api_headers()) as response:
243243
if response.status == 200:
244-
tags = await response.json()
244+
tags: list[dict[str, Any]] = await response.json()
245245
assert isinstance(tags, list)
246246
return GithubInfo(repo_path=url_path, tags=tags)
247247
return None
@@ -266,7 +266,7 @@ async def get_diff_info(
266266
if github_info is None:
267267
return None
268268

269-
versions_to_tags = {}
269+
versions_to_tags: dict[packaging.version.Version, str] = {}
270270
for tag in github_info.tags:
271271
tag_name = tag["name"]
272272
# Some packages in typeshed (e.g. emoji) have tag names
@@ -378,7 +378,7 @@ def describe_typeshed_files_modified(self) -> str:
378378
return analysis
379379

380380
def __str__(self) -> str:
381-
data_points = []
381+
data_points: list[str] = []
382382
if self.runtime_definitely_has_consistent_directory_structure_with_typeshed:
383383
data_points += [
384384
self.describe_public_files_added(),
@@ -398,7 +398,7 @@ async def analyze_diff(
398398
url = f"https://api.github.com/repos/{github_repo_path}/compare/{old_tag}...{new_tag}"
399399
async with session.get(url, headers=get_github_api_headers()) as response:
400400
response.raise_for_status()
401-
json_resp = await response.json()
401+
json_resp: dict[str, list[FileInfo]] = await response.json()
402402
assert isinstance(json_resp, dict)
403403
# https://docs.github.com/en/rest/commits/commits#compare-two-commits
404404
py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"]
@@ -581,7 +581,11 @@ def get_update_pr_body(update: Update, metadata: dict[str, Any]) -> str:
581581
if update.diff_analysis is not None:
582582
body += f"\n\n{update.diff_analysis}"
583583

584-
stubtest_will_run = not metadata.get("tool", {}).get("stubtest", {}).get("skip", False)
584+
# Loss of type due to infered [dict[Unknown, Unknown]]
585+
# scripts/stubsabot.py can't import tests/parse_metadata
586+
stubtest_will_run = (
587+
not metadata.get("tool", {}).get("stubtest", {}).get("skip", False) # pyright: ignore[reportUnknownMemberType]
588+
)
585589
if stubtest_will_run:
586590
body += textwrap.dedent(
587591
"""
@@ -611,10 +615,13 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession
611615
branch_name = f"{BRANCH_PREFIX}/{normalize(update.distribution)}"
612616
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
613617
with open(update.stub_path / "METADATA.toml", "rb") as f:
614-
meta = tomlkit.load(f)
618+
# tomlkit.load has partially unknown IO type
619+
# https://github.com/sdispater/tomlkit/pull/272
620+
meta = tomlkit.load(f) # pyright: ignore[reportUnknownMemberType]
615621
meta["version"] = update.new_version_spec
616622
with open(update.stub_path / "METADATA.toml", "w", encoding="UTF-8") as f:
617-
tomlkit.dump(meta, f)
623+
# tomlkit.dump has partially unknown IO type
624+
tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType]
618625
body = get_update_pr_body(update, meta)
619626
subprocess.check_call(["git 8000 ", "commit", "--all", "-m", f"{title}\n\n{body}"])
620627
if action_level <= ActionLevel.local:
@@ -637,12 +644,15 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS
637644
branch_name = f"{BRANCH_PREFIX}/{normalize(obsolete.distribution)}"
638645
subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/main"])
639646
with open(obsolete.stub_path / "METADATA.toml", "rb") as f:
640-
meta = tomlkit.load(f)
647+
# tomlkit.load has partially unknown IO type
648+
# https://github.com/sdispater/tomlkit/pull/272
649+
meta = tomlkit.load(f) # pyright: ignore[reportUnknownMemberType]
641650
obs_string = tomlkit.string(obsolete.obsolete_since_version)
642651
obs_string.comment(f"Released on {obsolete.obsolete_since_date.date().isoformat()}")
643652
meta["obsolete_since"] = obs_string
644653
with open(obsolete.stub_path / "METADATA.toml", "w", encoding="UTF-8") as f:
645-
tomlkit.dump(meta, f)
654+
# tomlkit.dump has partially unknown Mapping type
655+
tomlkit.dump(meta, f) # pyright: ignore[reportUnknownMemberType]
646656
body = "\n".join(f"{k}: {v}" for k, v in obsolete.links.items())
647657
subprocess.check_call(["git", "commit", "--all", "-m", f"{title}\n\n{body}"])
648658
if action_level <= ActionLevel.local:
@@ -727,7 +737,8 @@ async def main() -> None:
727737
if isinstance(update, Update):
728738
await suggest_typeshed_update(update, session, action_level=args.action_level)
729739
continue
730-
if isinstance(update, Obsolete):
740+
# Redundant, but keeping for extra runtime validation
741+
if isinstance(update, Obsolete): # pyright: ignore[reportUnnecessaryIsInstance]
731742
await suggest_typeshed_obsolete(update, session, action_level=args.action_level)
732743
continue
733744
except RemoteConflict as e:

‎tests/check_consistent.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import sys
1111
import urllib.parse
1212
from pathlib import Path
13+
from typing import TypedDict
1314

1415
import yaml
1516
from packaging.requirements import Requirement
@@ -93,7 +94,7 @@ def check_no_symlinks() -> None:
9394

9495

9596
def check_versions() -> None:
96-
versions = set()
97+
versions = set[str]()
9798
with open("stdlib/VERSIONS", encoding="UTF-8") as f:
9899
data = f.read().splitlines()
99100
for line in data:
@@ -115,7 +116,7 @@ def check_versions() -> None:
115116

116117

117118
def _find_stdlib_modules() -> set[str]:
118-
modules = set()
119+
modules = set[str]()
119120
for path, _, files in os.walk("stdlib"):
120121
for filename in files:
121122
base_module = ".".join(os.path.normpath(path).split(os.sep)[1:])
@@ -140,11 +141,21 @@ def get_txt_requirements() -> dict[str, SpecifierSet]:
140141
return {requirement.name: requirement.specifier for requirement in requirements}
141142

142143

144+
class PreCommitConfigRepos(TypedDict):
145+
hooks: list[dict[str, str]]
146+
repo: str
147+
rev: str
148+
149+
150+
class PreCommitConfig(TypedDict):
151+
repos: list[PreCommitConfigRepos]
152+
153+
143154
def get_precommit_requirements() -> dict[str, SpecifierSet]:
144155
with open(".pre-commit-config.yaml", encoding="UTF-8") as precommit_file:
145156
precommit = precommit_file.read()
146-
yam = yaml.load(precommit, Loader=yaml.Loader)
147-
precommit_requirements = {}
157+
yam: PreCommitConfig = yaml.load(precommit, Loader=yaml.Loader)
158+
precommit_requirements: dict[str, SpecifierSet] = {}
148159
for repo in yam["repos"]:
149160
if not repo.get("python_requirement", True):
150161
continue

‎tests/check_new_syntax.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010

1111
def check_new_syntax(tree: ast.AST, path: Path, stub: str) -> list[str]:
12-
errors = []
12+
errors: list[str] = []
1313

1414
class IfFinder(ast.NodeVisitor):
1515
def visit_If(self, node: ast.If) -> None:
@@ -31,7 +31,7 @@ def visit_If(self, node: ast.If) -> None:
3131

3232

3333
def main() -> None:
34-
errors = []
34+
errors: list[str] = []
3535
for path in chain(Path("stdlib").rglob("*.pyi"), Path("stubs").rglob("*.pyi")):
3636
with open(path, encoding="UTF-8") as f:
3737
stub = f.read()

‎tests/mypy_test.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242

4343
# Fail early if mypy isn't installed
4444
try:
45-
import mypy # noqa: F401
45+
import mypy # pyright: ignore[reportUnusedImport] # noqa: F401
4646
except ImportError:
4747
print_error("Cannot import mypy. Did you install it?")
4848
sys.exit(1)
@@ -57,7 +57,8 @@
5757
Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
5858

5959

60-
class CommandLineArgs(argparse.Namespace):
60+
@dataclass(init=False)
61+
class CommandLineArgs:
6162
verbose: int
6263
filter: list[Path]
6364
exclude: list[Path] | None
@@ -158,7 +159,7 @@ def match(path: Path, args: TestConfig) -> bool:
158159

159160

160161
def parse_versions(fname: StrPath) -> dict[str, tuple[VersionTuple, VersionTuple]]:
161-
result = {}
162+
result: dict[str, tuple[VersionTuple, VersionTuple]] = {}
162163
with open(fname, encoding="UTF-8") as f:
163164
for line in f:
164165
line = strip_comments(line)
@@ -209,7 +210,8 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) ->
209210
with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
210211
data = tomli.load(f)
211212

212-
mypy_tests_conf = data.get("mypy-tests")
213+
# TODO: This could be added to parse_metadata.py, but is currently unused
214+
mypy_tests_conf: dict[str, dict[str, Any]] = data.get("mypy-tests", {})
213215
if not mypy_tests_conf:
214216
return
215217

@@ -221,8 +223,8 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) ->
221223
assert module_name is not None, f"{section_name} should have a module_name key"
222224
assert isinstance(module_name, str), f"{section_name} should be a key-value pair"
223225

224-
values = mypy_section.get("values")
225-
assert values is not None, f"{section_name} should have a values section"
226+
assert "values" in mypy_section, f"{section_name} should have a values section"
227+
values: dict[str, dict[str, Any]] = mypy_section["values"]
226228
assert isinstance(values, dict), "values should be a section"
227229

228230
configurations.append(MypyDistConf(module_name, values.copy()))

‎tests/parse_metadata.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# This module is made specifically to abstract away those type errors
2+
# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownMemberType=false
3+
14
"""Tools to help parse and validate information stored in METADATA.toml files."""
25
from __future__ import annotations
36

@@ -188,7 +191,8 @@ def read_metadata(distribution: str) -> StubMetadata:
188191
uploaded_to_pypi = data.get("upload", True)
189192
assert type(uploaded_to_pypi) is bool
190193

191-
tools_settings = data.get("tool", {})
194+
empty_tools: dict[str, dict[str, object]] = {}
195+
tools_settings = data.get("tool", empty_tools)
192196
assert isinstance(tools_settings, dict)
193197
assert tools_settings.keys() <= _KNOWN_METADATA_TOOL_FIELDS.keys(), f"Unrecognised tool for {distribution!r}"
194198
for tool, tk in _KNOWN_METADATA_TOOL_FIELDS.items():
@@ -234,7 +238,8 @@ def read_dependencies(distribution: str) -> PackageDependencies:
234238
If a typeshed stub is removed, this function will consider it to be an external dependency.
235239
"""
236240
pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
237-
typeshed, external = [], []
241+
typeshed: list[str] = []
242+
external: list[str] = []
238243
for dependency in read_metadata(distribution).requires:
239244
maybe_typeshed_dependency = Requirement(dependency).name
240245
if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping:

‎tests/pytype_test.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#!/usr/bin/env python3
2+
# Lack of pytype typing
3+
# pyright: reportUnknownVariableType=false, reportUnknownMemberType=false, reportUnknownArgumentType=false, reportMissingTypeStubs=false
24
"""Test runner for typeshed.
35
46
Depends on pytype being installed.
@@ -19,11 +21,14 @@
1921
from collections.abc import Iterable, Sequence
2022

2123
import pkg_resources
22-
from pytype import config as pytype_config, load_pytd # type: ignore[import]
23-
from pytype.imports import typeshed # type: ignore[import]
2424

2525
from parse_metadata import read_dependencies
2626

27+
assert sys.platform != "win32"
28+
# pytype is not py.typed https://github.com/google/pytype/issues/1325
29+
from pytype import config as pytype_config, load_pytd # type: ignore[import] # noqa: E402
30+
from pytype.imports import typeshed # type: ignore[import] # noqa: E402
31+
2732
TYPESHED_SUBDIRS = ["stdlib", "stubs"]
2833
TYPESHED_HOME = "TYPESHED_HOME"
2934
_LOADERS = {}
@@ -155,7 +160,11 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]:
155160
for distribution in stub_distributions:
156161
for pkg in read_dependencies(distribution).external_pkgs:
157162
# See https://stackoverflow.com/a/54853084
158-
top_level_file = os.path.join(pkg_resources.get_distribution(pkg).egg_info, "top_level.txt") # type: ignore[attr-defined]
163+
top_level_file = os.path.join(
164+
# Fixed in #9747
165+
pkg_resources.get_distribution(pkg).egg_info, # type: ignore[attr-defined] # pyright: ignore[reportGeneralTypeIssues]
166+
"top_level.txt",
167+
)
159168
with open(top_level_file) as f:
160169
missing_modules.update(f.read().splitlines())
161170
return missing_modules

0 commit comments

Comments
 (0)
0